From 99806414be3133a5e8a7a9f3f6b2a14d65f4f817 Mon Sep 17 00:00:00 2001 From: Carter Date: Sat, 11 Nov 2023 13:57:38 -0800 Subject: [PATCH] Add e2e2 (#370) --- e2e2/.dockerignore | 1 + e2e2/.gitignore | 4 + e2e2/Makefile | 4 + e2e2/go.mod | 83 + e2e2/go.sum | 299 + e2e2/internal/framework_extensions/client.go | 136 + .../framework_extensions/conditions.go | 43 + e2e2/internal/framework_extensions/doc.go | 2 + e2e2/scripts/go-test.sh | 17 + e2e2/test/cases/nvidia/main_test.go | 82 + .../nvidia/manifests/efa-device-plugin.yaml | 94 + .../mpi-job-pytorch-training-multi-node.yaml | 110 + .../mpi-job-pytorch-training-single-node.yaml | 54 + .../cases/nvidia/manifests/mpi-operator.yaml | 8661 +++++++++++++++++ .../manifests/nvidia-device-plugin.yaml | 56 + e2e2/test/cases/nvidia/mpi_test.go | 124 + 16 files changed, 9770 insertions(+) create mode 100644 e2e2/.dockerignore create mode 100644 e2e2/.gitignore create mode 100644 e2e2/Makefile create mode 100644 e2e2/go.mod create mode 100644 e2e2/go.sum create mode 100644 e2e2/internal/framework_extensions/client.go create mode 100644 e2e2/internal/framework_extensions/conditions.go create mode 100644 e2e2/internal/framework_extensions/doc.go create mode 100755 e2e2/scripts/go-test.sh create mode 100644 e2e2/test/cases/nvidia/main_test.go create mode 100644 e2e2/test/cases/nvidia/manifests/efa-device-plugin.yaml create mode 100644 e2e2/test/cases/nvidia/manifests/mpi-job-pytorch-training-multi-node.yaml create mode 100644 e2e2/test/cases/nvidia/manifests/mpi-job-pytorch-training-single-node.yaml create mode 100644 e2e2/test/cases/nvidia/manifests/mpi-operator.yaml create mode 100644 e2e2/test/cases/nvidia/manifests/nvidia-device-plugin.yaml create mode 100644 e2e2/test/cases/nvidia/mpi_test.go diff --git a/e2e2/.dockerignore b/e2e2/.dockerignore new file mode 100644 index 000000000..d3fe3925c --- /dev/null +++ b/e2e2/.dockerignore @@ -0,0 +1 @@ +_bin/ \ No newline at end of file diff --git a/e2e2/.gitignore b/e2e2/.gitignore new file mode 100644 index 000000000..667d43091 --- /dev/null +++ b/e2e2/.gitignore @@ -0,0 +1,4 @@ +_bin +_rundir +_artifacts +.kuberlr diff --git a/e2e2/Makefile b/e2e2/Makefile new file mode 100644 index 000000000..c895fe0a8 --- /dev/null +++ b/e2e2/Makefile @@ -0,0 +1,4 @@ +BIN_DIR ?= _bin + +$(BIN_DIR)/runner: runner.go + go build -o $(BIN_DIR)/runner runner.go diff --git a/e2e2/go.mod b/e2e2/go.mod new file mode 100644 index 000000000..1650d4cff --- /dev/null +++ b/e2e2/go.mod @@ -0,0 +1,83 @@ +module github.com/aws/aws-k8s-tester/e2e2 + +go 1.21 + +require ( + k8s.io/api v0.28.3 + k8s.io/apimachinery v0.28.3 + k8s.io/cli-runtime v0.28.3 + k8s.io/client-go v0.28.3 + sigs.k8s.io/e2e-framework v0.3.0 +) + +require ( + github.com/aws/aws-sdk-go-v2 v1.22.0 // indirect + github.com/aws/aws-sdk-go-v2/config v1.20.0 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.14.0 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.4.0 // indirect + github.com/aws/aws-sdk-go-v2/service/eks v1.31.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.16.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.18.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.24.0 // indirect + github.com/aws/smithy-go v1.16.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.10.2 // indirect + github.com/evanphx/json-patch v4.12.0+incompatible // indirect + github.com/evanphx/json-patch/v5 v5.6.0 // indirect + github.com/go-errors/errors v1.4.2 // indirect + github.com/go-logr/logr v1.2.4 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.3 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.5.9 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/imdario/mergo v0.3.15 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/kubeflow/common v0.4.6 // indirect + github.com/kubeflow/mpi-operator v0.4.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/moby/spdystream v0.2.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/testify v1.8.4 // indirect + github.com/vladimirvivien/gexe v0.2.0 // indirect + github.com/xlab/treeprint v1.2.0 // indirect + go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/oauth2 v0.8.0 // indirect + golang.org/x/sync v0.2.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect + golang.org/x/time v0.3.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.30.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/klog/v2 v2.100.1 // indirect + k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect + k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect + sigs.k8s.io/controller-runtime v0.15.1 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect + sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect +) diff --git a/e2e2/go.sum b/e2e2/go.sum new file mode 100644 index 000000000..c9cbfa43d --- /dev/null +++ b/e2e2/go.sum @@ -0,0 +1,299 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/aws/aws-sdk-go-v2 v1.22.0 h1:CpTS3XO3MWNel8ohoazkLZC6scvkYL2k+m0yzFJ17Hg= +github.com/aws/aws-sdk-go-v2 v1.22.0/go.mod h1:Kd0OJtkW3Q0M0lUWGszapWjEvrXDzRW+D21JNsroB+c= +github.com/aws/aws-sdk-go-v2/config v1.20.0 h1:q2+/mqFhY0J9m3Tb5RGFE3R4sdaUkIe4k2EuDfE3c08= +github.com/aws/aws-sdk-go-v2/config v1.20.0/go.mod h1:7+1riCZXyT+sAGvneR5j+Zl1GyfbBUNQurpQTE6FP6k= +github.com/aws/aws-sdk-go-v2/credentials v1.14.0 h1:LQquqPE7cL55RQmA/UBoBKehDlEtMnQKm3B0Q672ePE= +github.com/aws/aws-sdk-go-v2/credentials v1.14.0/go.mod h1:q/3oaTPlamrQWHPwJe56Mjq9g1TYDgddvgTgWJtHTmE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.0 h1:lF/cVllNAPKgjDwN2RsQUX9g/f6hXer9f10ubLFSoug= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.0/go.mod h1:c28nJNzMVVb9TQpZ5q4tzZvwEJwf/7So7Ie2s90l1Fw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.0 h1:tN6dNNE4SzMuyMnVtQJXGVKX177/d5Zy4MuA1HA4KUc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.0/go.mod h1:F6MXWETIeetAHwFHyoHEqrcB3NpijFv9nLP5h9CXtT0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.0 h1:bfdsbTARDjaC/dSYGMO+E0psxFU4hTvCLnqYAfZ3D38= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.0/go.mod h1:Jg8XVv5M2V2wiAMvBFx+O59jg6Yr8vhP0bgNF/IuquM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.4.0 h1:21tlTXq3ev10yLMAjXZzpkZbrl49h3ElSjmxD57tD/E= +github.com/aws/aws-sdk-go-v2/internal/ini v1.4.0/go.mod h1:d9YrBHJhyzDCv5UsEVRizHlFV6Q0sLemFq6uxuqWfUw= +github.com/aws/aws-sdk-go-v2/service/eks v1.31.0 h1:OpUXxM82JiAjayWyUMstydx7XsQpTjvBcf3LFy/Iv1s= +github.com/aws/aws-sdk-go-v2/service/eks v1.31.0/go.mod h1:LndnphC2kK40Q/drKD0zLFDK5INeKhSzyzmH9nyQVnk= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.0 h1:dJnwy5Awv+uvfk73aRENVbv1cSQQ60ydCkPaun097KM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.0/go.mod h1:RsPWWy7u/hwmFX57sQ7MLvrvJeYyNkiMm5BaavpoU18= +github.com/aws/aws-sdk-go-v2/service/sso v1.16.0 h1:ZIlR6Wr/EgYwBdEz1NWBqdUsTh0mV7A68pId3YZl6H0= +github.com/aws/aws-sdk-go-v2/service/sso v1.16.0/go.mod h1:O7B5cpuhhJKefAKkM7onb0McmpHyKnsH4RrHJhOyq7M= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.18.0 h1:3BZyJei4k1SHdSAFhg9Qg15NnG3v5zosZyFWPm7df/A= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.18.0/go.mod h1:Td8EvzggonY02wLaqSpwybI3GbmA0PWoprKGil2uwJg= +github.com/aws/aws-sdk-go-v2/service/sts v1.24.0 h1:f/V5Y9OaHuNRrA9MntNQNAtMFXqhKj8HTEPnH81eXMI= +github.com/aws/aws-sdk-go-v2/service/sts v1.24.0/go.mod h1:HnCUMNz2XqwnEEk5X6oeDYB2HgOLFpJ/LyfilN8WErs= +github.com/aws/smithy-go v1.16.0 h1:gJZEH/Fqh+RsvlJ1Zt4tVAtV6bKkp3cC+R6FCZMNzik= +github.com/aws/smithy-go v1.16.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.10.2 h1:hIovbnmBTLjHXkqEBUz3HGpXZdM7ZrE9fJIZIqlJLqE= +github.com/emicklei/go-restful/v3 v3.10.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= +github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= +github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kubeflow/common v0.4.6 h1:yzJf/HEdS6ginD0GlVkgbOFie0Sp66VdGjXidAGZIlk= +github.com/kubeflow/common v0.4.6/go.mod h1:43MAof/uhpJA2C0urynqatE3oKFQc7m2HLmJty7waqY= +github.com/kubeflow/mpi-operator v0.4.0 h1:PS4jLoMuRyrk/DHuYkI0D46sQQYpQt375HjOV4KVMFs= +github.com/kubeflow/mpi-operator v0.4.0/go.mod h1:/A4mTy/RYh2UIgaGUiXUaW70eThjsogu80WbbcZpuMg= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= +github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/vladimirvivien/gexe v0.2.0 h1:nbdAQ6vbZ+ZNsolCgSVb9Fno60kzSuvtzVh6Ytqi/xY= +github.com/vladimirvivien/gexe v0.2.0/go.mod h1:LHQL00w/7gDUKIak24n801ABp8C+ni6eBht9vGVst8w= +github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= +github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY= +go.starlark.net v0.0.0-20230525235612-a134d8f9ddca/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= +golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.3.0 h1:8NFhfS6gzxNqjLIYnZxg319wZ5Qjnx4m/CcX+Klzazc= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/api v0.28.3 h1:Gj1HtbSdB4P08C8rs9AR94MfSGpRhJgsS+GF9V26xMM= +k8s.io/api v0.28.3/go.mod h1:MRCV/jr1dW87/qJnZ57U5Pak65LGmQVkKTzf3AtKFHc= +k8s.io/apiextensions-apiserver v0.27.2 h1:iwhyoeS4xj9Y7v8YExhUwbVuBhMr3Q4bd/laClBV6Bo= +k8s.io/apimachinery v0.28.3 h1:B1wYx8txOaCQG0HmYF6nbpU8dg6HvA06x5tEffvOe7A= +k8s.io/apimachinery v0.28.3/go.mod h1:uQTKmIqs+rAYaq+DFaoD2X7pcjLOqbQX2AOiO0nIpb8= +k8s.io/cli-runtime v0.28.3 h1:lvuJYVkwCqHEvpS6KuTZsUVwPePFjBfSGvuaLl2SxzA= +k8s.io/cli-runtime v0.28.3/go.mod h1:jeX37ZPjIcENVuXDDTskG3+FnVuZms5D9omDXS/2Jjc= +k8s.io/client-go v0.28.3 h1:2OqNb72ZuTZPKCl+4gTKvqao0AMOl9f3o2ijbAj3LI4= +k8s.io/client-go v0.28.3/go.mod h1:LTykbBp9gsA7SwqirlCXBWtK0guzfhpoW4qSm7i9dxo= +k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= +k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= +k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.15.1 h1:9UvgKD4ZJGcj24vefUFgZFP3xej/3igL9BsOUTb/+4c= +sigs.k8s.io/controller-runtime v0.15.1/go.mod h1:7ngYvp1MLT+9GeZ+6lH3LOlcHkp/+tzA/fmHa4iq9kk= +sigs.k8s.io/e2e-framework v0.3.0 h1:eqQALBtPCth8+ulTs6lcPK7ytV5rZSSHJzQHZph4O7U= +sigs.k8s.io/e2e-framework v0.3.0/go.mod h1:C+ef37/D90Dc7Xq1jQnNbJYscrUGpxrWog9bx2KIa+c= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 h1:XX3Ajgzov2RKUdc5jW3t5jwY7Bo7dcRm+tFxT+NfgY0= +sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3/go.mod h1:9n16EZKMhXBNSiUC5kSdFQJkdH3zbxS/JoO619G1VAY= +sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 h1:W6cLQc5pnqM7vh3b7HvGNfXrJ/xL6BDMS0v1V/HHg5U= +sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3/go.mod h1:JWP1Fj0VWGHyw3YUPjXSQnRnrwezrZSrApfX5S0nIag= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/e2e2/internal/framework_extensions/client.go b/e2e2/internal/framework_extensions/client.go new file mode 100644 index 000000000..01df70414 --- /dev/null +++ b/e2e2/internal/framework_extensions/client.go @@ -0,0 +1,136 @@ +package frameworkext + +import ( + "context" + "os" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/cli-runtime/pkg/resource" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" + "sigs.k8s.io/e2e-framework/klient/decoder" + "sigs.k8s.io/e2e-framework/klient/k8s" +) + +// ApplyFiles creates Kubernetes objects contained in manifest file(s), in a manner similar to `kubectl apply -f` +// Multiple objects may be in each manifest file. +// The manifest files are processed in order. +func ApplyFiles(restConfig *rest.Config, manifestFiles []string) error { + for _, manifestFile := range manifestFiles { + if err := ApplyFile(restConfig, manifestFile); err != nil { + return err + } + } + return nil +} + +// ApplyFile creates Kubernetes objects contained in a manifest file, in a manner similar to `kubectl apply -f` +// Multiple objects may be in the manifest file. +func ApplyFile(restConfig *rest.Config, manifestFile string) error { + f, err := os.Open(manifestFile) + if err != nil { + return err + } + objs, err := decoder.DecodeAll(context.TODO(), f) + if err != nil { + return err + } + return processObjects(restConfig, objs, func(client *resource.Helper, obj k8s.Object) error { + namespace, err := meta.NewAccessor().Namespace(obj) + if err != nil { + return err + } + if namespace == "" { + namespace = "default" + } + _, err = client.Create(namespace, false, obj) + return err + }) +} + +// DeleteFiles deletes Kubernetes objects contained in manifest file(s), in a manner similar to `kubectl delete -f` +// Multiple objects may be in each manifest file. +// The manifest files are processed in order. +func DeleteFiles(restConfig *rest.Config, manifestFiles []string) error { + for _, manifestFile := range manifestFiles { + if err := DeleteFile(restConfig, manifestFile); err != nil { + return err + } + } + return nil +} + +// DeleteFile deletes Kubernetes objects contained in a manifest file, in a manner similar to `kubectl delete -f` +// Multiple objects may be in the manifest file. +func DeleteFile(restConfig *rest.Config, manifestFile string) error { + f, err := os.Open(manifestFile) + if err != nil { + return err + } + objs, err := decoder.DecodeAll(context.TODO(), f) + if err != nil { + return err + } + return processObjects(restConfig, objs, func(client *resource.Helper, obj k8s.Object) error { + name, err := meta.NewAccessor().Name(obj) + if err != nil { + return err + } + namespace, err := meta.NewAccessor().Namespace(obj) + if err != nil { + return err + } + if namespace == "" { + namespace = "default" + } + _, err = client.Delete(namespace, name) + return err + }) +} + +// processObjects applies a processFunc to each object, supplying it a dynamically-typed client appropriate for the object +func processObjects(restConfig *rest.Config, objs []k8s.Object, processFunc func(client *resource.Helper, obj k8s.Object) error) error { + clientset, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return err + } + groupResources, err := restmapper.GetAPIGroupResources(clientset.Discovery()) + if err != nil { + return err + } + rm := restmapper.NewDiscoveryRESTMapper(groupResources) + for _, obj := range objs { + client, err := newResourceHelper(restConfig, rm, obj) + if err != nil { + return err + } + processFunc(client, obj) + } + return nil +} + +func newResourceHelper(restConfig *rest.Config, rm meta.RESTMapper, obj runtime.Object) (*resource.Helper, error) { + gvk := obj.GetObjectKind().GroupVersionKind() + gk := schema.GroupKind{Group: gvk.Group, Kind: gvk.Kind} + mapping, err := rm.RESTMapping(gk, gvk.Version) + if err != nil { + return nil, err + } + gv := mapping.GroupVersionKind.GroupVersion() + restConfig.ContentConfig = resource.UnstructuredPlusDefaultContentConfig() + restConfig.GroupVersion = &gv + if len(gv.Group) == 0 { + restConfig.APIPath = "/api" + } else { + restConfig.APIPath = "/apis" + } + restClient, err := rest.RESTClientFor(restConfig) + if err != nil { + return nil, err + } + + return resource.NewHelper(restClient, mapping), nil +} diff --git a/e2e2/internal/framework_extensions/conditions.go b/e2e2/internal/framework_extensions/conditions.go new file mode 100644 index 000000000..c23e50534 --- /dev/null +++ b/e2e2/internal/framework_extensions/conditions.go @@ -0,0 +1,43 @@ +package frameworkext + +import ( + "context" + + appsv1 "k8s.io/api/apps/v1" + apimachinerywait "k8s.io/apimachinery/pkg/util/wait" + + "sigs.k8s.io/e2e-framework/klient/k8s" + "sigs.k8s.io/e2e-framework/klient/k8s/resources" +) + +type ConditionExtension struct { + resources *resources.Resources +} + +func NewConditionExtension(r *resources.Resources) *ConditionExtension { + return &ConditionExtension{resources: r} +} + +// ResourceMatch is a helper function used to check if the resource under question has met a pre-defined state. This can +// be leveraged for checking fields on a resource that may not be immediately present upon creation. +func (c *ConditionExtension) ResourceMatch(obj k8s.Object, matchFetcher func(object k8s.Object) bool) apimachinerywait.ConditionWithContextFunc { + return func(ctx context.Context) (done bool, err error) { + if err := c.resources.Get(ctx, obj.GetName(), obj.GetNamespace(), obj); err != nil { + return false, err + } + return matchFetcher(obj), nil + } +} + +func (c *ConditionExtension) DaemonSetReady(daemonset k8s.Object) apimachinerywait.ConditionWithContextFunc { + return func(ctx context.Context) (done bool, err error) { + if err := c.resources.Get(ctx, daemonset.GetName(), daemonset.GetNamespace(), daemonset); err != nil { + return false, err + } + status := daemonset.(*appsv1.DaemonSet).Status + if status.NumberReady == status.DesiredNumberScheduled && status.NumberUnavailable == 0 { + done = true + } + return + } +} diff --git a/e2e2/internal/framework_extensions/doc.go b/e2e2/internal/framework_extensions/doc.go new file mode 100644 index 000000000..c0f165f2b --- /dev/null +++ b/e2e2/internal/framework_extensions/doc.go @@ -0,0 +1,2 @@ +// Package frameworkext contains extensions to sigs.k8s.io/e2e-framework +package frameworkext diff --git a/e2e2/scripts/go-test.sh b/e2e2/scripts/go-test.sh new file mode 100755 index 000000000..78ff27c99 --- /dev/null +++ b/e2e2/scripts/go-test.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail +set -o nounset +set -o xtrace + +if [ "$#" -eq 0 ]; then + echo "Usage: $0 TEST_PATTERN [TEST_FLAGS]" + exit 1 +fi + +TEST_PATTERN="${1}" +shift + +# -count=1 is not a cacheable flag, so the test will always be executed +go test --timeout 0 -v -count=1 "${TEST_PATTERN}" ${@} diff --git a/e2e2/test/cases/nvidia/main_test.go b/e2e2/test/cases/nvidia/main_test.go new file mode 100644 index 000000000..ad6c72c4a --- /dev/null +++ b/e2e2/test/cases/nvidia/main_test.go @@ -0,0 +1,82 @@ +package nvidia + +import ( + "context" + "log" + "os" + "slices" + "testing" + "time" + + fwext "github.com/aws/aws-k8s-tester/e2e2/internal/framework_extensions" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/e2e-framework/klient/wait" + "sigs.k8s.io/e2e-framework/klient/wait/conditions" + "sigs.k8s.io/e2e-framework/pkg/env" + "sigs.k8s.io/e2e-framework/pkg/envconf" +) + +var ( + testenv env.Environment +) + +func TestMain(m *testing.M) { + cfg, err := envconf.NewFromFlags() + if err != nil { + log.Fatalf("failed to initialize test environment: %v", err) + } + testenv = env.NewWithConfig(cfg) + + // all NVIDIA tests require the device plugin and MPI operator + manifests := []string{ + "manifests/nvidia-device-plugin.yaml", + "manifests/mpi-operator.yaml", + } + + testenv.Setup( + func(ctx context.Context, config *envconf.Config) (context.Context, error) { + err := fwext.ApplyFiles(config.Client().RESTConfig(), manifests) + if err != nil { + return ctx, err + } + return ctx, nil + }, + func(ctx context.Context, config *envconf.Config) (context.Context, error) { + dep := appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "mpi-operator", Namespace: "mpi-operator"}, + } + err := wait.For(conditions.New(config.Client().Resources()).DeploymentConditionMatch(&dep, appsv1.DeploymentAvailable, v1.ConditionTrue), + wait.WithTimeout(time.Minute*5)) + if err != nil { + return ctx, err + } + return ctx, nil + }, + func(ctx context.Context, config *envconf.Config) (context.Context, error) { + ds := appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Name: "nvidia-device-plugin-daemonset", Namespace: "kube-system"}, + } + err := wait.For(fwext.NewConditionExtension(config.Client().Resources()).DaemonSetReady(&ds), + wait.WithTimeout(time.Minute*5)) + if err != nil { + return ctx, err + } + return ctx, nil + }, + ) + + testenv.Finish( + func(ctx context.Context, config *envconf.Config) (context.Context, error) { + slices.Reverse(manifests) + err := fwext.DeleteFiles(config.Client().RESTConfig(), manifests) + if err != nil { + return ctx, err + } + return ctx, nil + }, + ) + + os.Exit(testenv.Run(m)) +} diff --git a/e2e2/test/cases/nvidia/manifests/efa-device-plugin.yaml b/e2e2/test/cases/nvidia/manifests/efa-device-plugin.yaml new file mode 100644 index 000000000..774ea1df5 --- /dev/null +++ b/e2e2/test/cases/nvidia/manifests/efa-device-plugin.yaml @@ -0,0 +1,94 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: efa-device-plugin-daemonset + namespace: kube-system +spec: + selector: + matchLabels: + name: aws-efa-k8s-device-plugin + updateStrategy: + type: RollingUpdate + template: + metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + labels: + name: aws-efa-k8s-device-plugin + spec: + serviceAccount: default + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: aws.amazon.com/efa + operator: Exists + effect: NoSchedule + priorityClassName: "system-node-critical" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "beta.kubernetes.io/instance-type" + operator: In + values: + - c5n.18xlarge + - c5n.metal + - g4dn.metal + - c5n.9xlarge + - i3en.24xlarge + - i3en.metal + - inf1.24xlarge + - m5dn.24xlarge + - m5n.24xlarge + - p3dn.24xlarge + - r5dn.24xlarge + - r5n.24xlarge + - p4d.24xlarge + - hpc6a.48xlarge + - dl1.24xlarge + - g5.48xlarge + - p4de.24xlarge + - trn1.32xlarge + - trn1n.32xlarge + - matchExpressions: + - key: "node.kubernetes.io/instance-type" + operator: In + values: + - c5n.18xlarge + - c5n.metal + - c5n.9xlarge + - g4dn.metal + - i3en.24xlarge + - i3en.metal + - inf1.24xlarge + - m5dn.24xlarge + - m5n.24xlarge + - p3dn.24xlarge + - r5dn.24xlarge + - r5n.24xlarge + - p4d.24xlarge + - hpc6a.48xlarge + - dl1.24xlarge + - g5.48xlarge + - p4de.24xlarge + - trn1.32xlarge + - trn1n.32xlarge + hostNetwork: true + containers: + - image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/eks/aws-efa-k8s-device-plugin:v0.3.3 + imagePullPolicy: Always + name: aws-efa-k8s-device-plugin + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: false + volumeMounts: + - name: device-plugin + mountPath: /var/lib/kubelet/device-plugins + volumes: + - name: device-plugin + hostPath: + path: /var/lib/kubelet/device-plugins diff --git a/e2e2/test/cases/nvidia/manifests/mpi-job-pytorch-training-multi-node.yaml b/e2e2/test/cases/nvidia/manifests/mpi-job-pytorch-training-multi-node.yaml new file mode 100644 index 000000000..aa92d9c28 --- /dev/null +++ b/e2e2/test/cases/nvidia/manifests/mpi-job-pytorch-training-multi-node.yaml @@ -0,0 +1,110 @@ +apiVersion: kubeflow.org/v2beta1 +kind: MPIJob +metadata: + name: pytorch-training-multi-node +spec: + slotsPerWorker: 8 + runPolicy: + cleanPodPolicy: Running + mpiReplicaSpecs: + Launcher: + replicas: 1 + template: + spec: + restartPolicy: OnFailure + initContainers: + - image: public.ecr.aws/w6p6i9i7/aws-efa-nccl-rdma:22.03-pt-py3 + name: init + command: ["sh", "-c", "sleep 5"] + containers: + - image: public.ecr.aws/w6p6i9i7/aws-efa-nccl-rdma:22.03-pt-py3 + imagePullPolicy: Always + name: nccl-test-launcher + env: + - name: LD_LIBRARY_PATH + value: /opt/amazon/openmpi/lib:/opt/nccl/build/lib:/opt/amazon/efa/lib:/opt/aws-ofi-nccl/install/lib:/usr/local/nvidia/lib:$LD_LIBRARY_PATH + - name: PATH + value: $PATH:/opt/amazon/efa/bin:/usr/sbin:/usr/bin:/usr/local/bin + - name: XLA_FLAGS + value: "--xla_gpu_cuda_data_dir=/usr/local/cuda" + - name: TF_XLA_FLAGS + value: "--tf_xla_cpu_global_jit" + command: + - /opt/amazon/openmpi/bin/mpirun + - --allow-run-as-root + - --tag-output + - -np + - "16" + - -bind-to + - none + - -map-by + - slot + - -x + - PATH + - -x + - LD_LIBRARY_PATH + - -x + - XLA_FLAGS + - -x + - TF_XLA_FLAGS + - -x + - NCCL_DEBUG=INFO + - -x + - NCCL_ALGO=RING + - -x + - RDMAV_FORK_SAFE=1 + - -x + - NCCL_PROTO=simple + - -x + - FI_LOG_LEVEL=warn + - -x + - FI_EFA_USE_DEVICE_RDMA=1 + - -x + - NCCL_PROTO=simple + - --mca + - pml + - ^cm + - --oversubscribe + - /opt/nccl-tests/build/all_reduce_perf + - -b + - "8" + - -e + - 2G + - -f + - "2" + - -t + - "1" + - -g + - "1" + - -c + - "1" + - -n + - "100" + Worker: + replicas: 2 + template: + spec: + volumes: + - name: dshm + emptyDir: + medium: Memory + nodeSelector: + beta.kubernetes.io/instance-type: p4d.24xlarge + containers: + - image: public.ecr.aws/w6p6i9i7/aws-efa-nccl-rdma:22.03-pt-py3 + imagePullPolicy: Always + name: nccl-worker + volumeMounts: + - mountPath: /dev/shm + name: dshm + resources: + limits: + nvidia.com/gpu: 8 + hugepages-2Mi: 5120Mi + vpc.amazonaws.com/efa: 4 + memory: 8000Mi + requests: + nvidia.com/gpu: 8 + hugepages-2Mi: 5120Mi + vpc.amazonaws.com/efa: 4 + memory: 8000Mi diff --git a/e2e2/test/cases/nvidia/manifests/mpi-job-pytorch-training-single-node.yaml b/e2e2/test/cases/nvidia/manifests/mpi-job-pytorch-training-single-node.yaml new file mode 100644 index 000000000..9284ec212 --- /dev/null +++ b/e2e2/test/cases/nvidia/manifests/mpi-job-pytorch-training-single-node.yaml @@ -0,0 +1,54 @@ +--- +# container image from: https://github.com/aws/deep-learning-containers/blob/master/available_images.md +apiVersion: kubeflow.org/v2beta1 +kind: MPIJob +metadata: + name: pytorch-training-single-node +spec: + slotsPerWorker: 4 + runPolicy: + cleanPodPolicy: Running + mpiImplementation: OpenMPI + mpiReplicaSpecs: + Launcher: + replicas: 1 + template: + spec: + restartPolicy: OnFailure + containers: + - image: 763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training:2.1.0-gpu-py310-cu121-ubuntu20.04-ec2 + name: gpu-test + command: + - mpirun + - --allow-run-as-root + - -np + - "1" + - -mca + - btl_tcp_if_exclude + - lo + - -mca + - pml + - ob1 + - -mca + - btl + - ^openib + - --bind-to + - none + - -map-by + - slot + - -x + - LD_LIBRARY_PATH + - -x + - PATH + - -x + - NCCL_SOCKET_IFNAME=eth0 + - -x + - NCCL_DEBUG=INFO + - -x + - MXNET_CUDNN_AUTOTUNE_DEFAULT=0 + - python + - -c + - import os; os.system("git clone https://github.com/pytorch/examples.git /pytorch-examples"); os.system("git -C pytorch-examples checkout 0f0c9131ca5c79d1332dce1f4c06fe942fbdc665"); os.system("python /pytorch-examples/mnist/main.py --epochs 3") + resources: + limits: + nvidia.com/gpu: 1 diff --git a/e2e2/test/cases/nvidia/manifests/mpi-operator.yaml b/e2e2/test/cases/nvidia/manifests/mpi-operator.yaml new file mode 100644 index 000000000..89689aa98 --- /dev/null +++ b/e2e2/test/cases/nvidia/manifests/mpi-operator.yaml @@ -0,0 +1,8661 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + labels: + app: mpi-operator + app.kubernetes.io/component: mpijob + app.kubernetes.io/name: mpi-operator + kustomize.component: mpi-operator + name: mpi-operator +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.12.0 + labels: + app: mpi-operator + app.kubernetes.io/component: mpijob + app.kubernetes.io/name: mpi-operator + kustomize.component: mpi-operator + name: mpijobs.kubeflow.org +spec: + group: kubeflow.org + names: + kind: MPIJob + listKind: MPIJobList + plural: mpijobs + singular: mpijob + scope: Namespaced + versions: + - name: v2beta1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + launcherCreationPolicy: + default: AtStartup + description: launcherCreationPolicy if WaitForWorkersReady, the launcher + is created only after all workers are in Ready state. Defaults to + AtStartup. + type: string + mpiImplementation: + default: OpenMPI + description: MPIImplementation is the MPI implementation. Options + are "OpenMPI" (default), "Intel" and "MPICH". + enum: + - OpenMPI + - Intel + - MPICH + type: string + mpiReplicaSpecs: + additionalProperties: + description: ReplicaSpec is a description of the replica + properties: + replicas: + description: Replicas is the desired number of replicas of the + given template. If unspecified, defaults to 1. + format: int32 + type: integer + restartPolicy: + description: Restart policy for all replicas within the job. + One of Always, OnFailure, Never and ExitCode. Default to Never. + type: string + template: + description: Template is the object that describes the pod that + will be created for this replica. RestartPolicy in PodTemplateSpec + will be overide by RestartPolicy in ReplicaSpec + properties: + metadata: + description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata' + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: 'Specification of the desired behavior of the + pod. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' + properties: + activeDeadlineSeconds: + description: Optional duration in seconds the pod may + be active on the node relative to StartTime before + the system will actively try to mark it failed and + kill associated containers. Value must be a positive + integer. + format: int64 + type: integer + affinity: + description: If specified, the pod's scheduling constraints + properties: + nodeAffinity: + description: Describes node affinity scheduling + rules for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule + pods to nodes that satisfy the affinity expressions + specified by this field, but it may choose + a node that violates one or more of the expressions. + The node that is most preferred is the one + with the greatest sum of weights, i.e. for + each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum + by iterating through the elements of this + field and adding "weight" to the sum if the + node matches the corresponding matchExpressions; + the node(s) with the highest sum are the most + preferred. + items: + description: An empty preferred scheduling + term matches all objects with implicit weight + 0 (i.e. it's a no-op). A null preferred + scheduling term matches no objects (i.e. + is also a no-op). + properties: + preference: + description: A node selector term, associated + with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: A node selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: Represents a key's + relationship to a set of values. + Valid operators are In, NotIn, + Exists, DoesNotExist. Gt, + and Lt. + type: string + values: + description: An array of string + values. If the operator is + In or NotIn, the values array + must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. + If the operator is Gt or Lt, + the values array must have + a single element, which will + be interpreted as an integer. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: A node selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: Represents a key's + relationship to a set of values. + Valid operators are In, NotIn, + Exists, DoesNotExist. Gt, + and Lt. + type: string + values: + description: An array of string + values. If the operator is + In or NotIn, the values array + must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. + If the operator is Gt or Lt, + the values array must have + a single element, which will + be interpreted as an integer. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching + the corresponding nodeSelectorTerm, + in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified + by this field are not met at scheduling time, + the pod will not be scheduled onto the node. + If the affinity requirements specified by + this field cease to be met at some point during + pod execution (e.g. due to an update), the + system may or may not try to eventually evict + the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector + terms. The terms are ORed. + items: + description: A null or empty node selector + term matches no objects. The requirements + of them are ANDed. The TopologySelectorTerm + type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: A node selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: Represents a key's + relationship to a set of values. + Valid operators are In, NotIn, + Exists, DoesNotExist. Gt, + and Lt. + type: string + values: + description: An array of string + values. If the operator is + In or NotIn, the values array + must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. + If the operator is Gt or Lt, + the values array must have + a single element, which will + be interpreted as an integer. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: A node selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: Represents a key's + relationship to a set of values. + Valid operators are In, NotIn, + Exists, DoesNotExist. Gt, + and Lt. + type: string + values: + description: An array of string + values. If the operator is + In or NotIn, the values array + must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. + If the operator is Gt or Lt, + the values array must have + a single element, which will + be interpreted as an integer. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules + (e.g. co-locate this pod in the same node, zone, + etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule + pods to nodes that satisfy the affinity expressions + specified by this field, but it may choose + a node that violates one or more of the expressions. + The node that is most preferred is the one + with the greatest sum of weights, i.e. for + each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum + by iterating through the elements of this + field and adding "weight" to the sum if the + node has pods which matches the corresponding + podAffinityTerm; the node(s) with the highest + sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added + per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: A label query over a + set of resources, in this case pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: A label selector + requirement is a selector + that contains values, a key, + and an operator that relates + the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: operator represents + a key's relationship to + a set of values. Valid + operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an + array of string values. + If the operator is In + or NotIn, the values array + must be non-empty. If + the operator is Exists + or DoesNotExist, the values + array must be empty. This + array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a + map of {key,value} pairs. A + single {key,value} in the matchLabels + map is equivalent to an element + of matchExpressions, whose key + field is "key", the operator + is "In", and the values array + contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the + set of namespaces that the term + applies to. The term is applied + to the union of the namespaces selected + by this field and the ones listed + in the namespaces field. null selector + and null or empty namespaces list + means "this pod's namespace". An + empty selector ({}) matches all + namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: A label selector + requirement is a selector + that contains values, a key, + and an operator that relates + the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: operator represents + a key's relationship to + a set of values. Valid + operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an + array of string values. + If the operator is In + or NotIn, the values array + must be non-empty. If + the operator is Exists + or DoesNotExist, the values + array must be empty. This + array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a + map of {key,value} pairs. A + single {key,value} in the matchLabels + map is equivalent to an element + of matchExpressions, whose key + field is "key", the operator + is "In", and the values array + contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies + a static list of namespace names + that the term applies to. The term + is applied to the union of the namespaces + listed in this field and the ones + selected by namespaceSelector. null + or empty namespaces list and null + namespaceSelector means "this pod's + namespace". + items: + type: string + type: array + topologyKey: + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where + co-located is defined as running + on a node whose value of the label + with key topologyKey matches that + of any node on which any of the + selected pods is running. Empty + topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching + the corresponding podAffinityTerm, in + the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified + by this field are not met at scheduling time, + the pod will not be scheduled onto the node. + If the affinity requirements specified by + this field cease to be met at some point during + pod execution (e.g. due to a pod label update), + the system may or may not try to eventually + evict the pod from its node. When there are + multiple elements, the lists of nodes corresponding + to each podAffinityTerm are intersected, i.e. + all terms must be satisfied. + items: + description: Defines a set of pods (namely + those matching the labelSelector relative + to the given namespace(s)) that this pod + should be co-located (affinity) or not co-located + (anti-affinity) with, where co-located is + defined as running on a node whose value + of the label with key matches + that of any node on which a pod of the set + of pods is running + properties: + labelSelector: + description: A label query over a set + of resources, in this case pods. + properties: + matchExpressions: + description: matchExpressions is a + list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: operator represents + a key's relationship to a + set of values. Valid operators + are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values + array must be non-empty. If + the operator is Exists or + DoesNotExist, the values array + must be empty. This array + is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map + of {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set + of namespaces that the term applies + to. The term is applied to the union + of the namespaces selected by this field + and the ones listed in the namespaces + field. null selector and null or empty + namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a + list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: operator represents + a key's relationship to a + set of values. Valid operators + are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values + array must be non-empty. If + the operator is Exists or + DoesNotExist, the values array + must be empty. This array + is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map + of {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static + list of namespace names that the term + applies to. The term is applied to the + union of the namespaces listed in this + field and the ones selected by namespaceSelector. + null or empty namespaces list and null + namespaceSelector means "this pod's + namespace". + items: + type: string + type: array + topologyKey: + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where co-located + is defined as running on a node whose + value of the label with key topologyKey + matches that of any node on which any + of the selected pods is running. Empty + topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling + rules (e.g. avoid putting this pod in the same + node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule + pods to nodes that satisfy the anti-affinity + expressions specified by this field, but it + may choose a node that violates one or more + of the expressions. The node that is most + preferred is the one with the greatest sum + of weights, i.e. for each node that meets + all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity + expressions, etc.), compute a sum by iterating + through the elements of this field and adding + "weight" to the sum if the node has pods which + matches the corresponding podAffinityTerm; + the node(s) with the highest sum are the most + preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added + per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: A label query over a + set of resources, in this case pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: A label selector + requirement is a selector + that contains values, a key, + and an operator that relates + the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: operator represents + a key's relationship to + a set of values. Valid + operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an + array of string values. + If the operator is In + or NotIn, the values array + must be non-empty. If + the operator is Exists + or DoesNotExist, the values + array must be empty. This + array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a + map of {key,value} pairs. A + single {key,value} in the matchLabels + map is equivalent to an element + of matchExpressions, whose key + field is "key", the operator + is "In", and the values array + contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the + set of namespaces that the term + applies to. The term is applied + to the union of the namespaces selected + by this field and the ones listed + in the namespaces field. null selector + and null or empty namespaces list + means "this pod's namespace". An + empty selector ({}) matches all + namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: A label selector + requirement is a selector + that contains values, a key, + and an operator that relates + the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: operator represents + a key's relationship to + a set of values. Valid + operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an + array of string values. + If the operator is In + or NotIn, the values array + must be non-empty. If + the operator is Exists + or DoesNotExist, the values + array must be empty. This + array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a + map of {key,value} pairs. A + single {key,value} in the matchLabels + map is equivalent to an element + of matchExpressions, whose key + field is "key", the operator + is "In", and the values array + contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies + a static list of namespace names + that the term applies to. The term + is applied to the union of the namespaces + listed in this field and the ones + selected by namespaceSelector. null + or empty namespaces list and null + namespaceSelector means "this pod's + namespace". + items: + type: string + type: array + topologyKey: + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where + co-located is defined as running + on a node whose value of the label + with key topologyKey matches that + of any node on which any of the + selected pods is running. Empty + topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching + the corresponding podAffinityTerm, in + the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the anti-affinity requirements + specified by this field are not met at scheduling + time, the pod will not be scheduled onto the + node. If the anti-affinity requirements specified + by this field cease to be met at some point + during pod execution (e.g. due to a pod label + update), the system may or may not try to + eventually evict the pod from its node. When + there are multiple elements, the lists of + nodes corresponding to each podAffinityTerm + are intersected, i.e. all terms must be satisfied. + items: + description: Defines a set of pods (namely + those matching the labelSelector relative + to the given namespace(s)) that this pod + should be co-located (affinity) or not co-located + (anti-affinity) with, where co-located is + defined as running on a node whose value + of the label with key matches + that of any node on which a pod of the set + of pods is running + properties: + labelSelector: + description: A label query over a set + of resources, in this case pods. + properties: + matchExpressions: + description: matchExpressions is a + list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: operator represents + a key's relationship to a + set of values. Valid operators + are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values + array must be non-empty. If + the operator is Exists or + DoesNotExist, the values array + must be empty. This array + is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map + of {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set + of namespaces that the term applies + to. The term is applied to the union + of the namespaces selected by this field + and the ones listed in the namespaces + field. null selector and null or empty + namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a + list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: operator represents + a key's relationship to a + set of values. Valid operators + are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values + array must be non-empty. If + the operator is Exists or + DoesNotExist, the values array + must be empty. This array + is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map + of {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static + list of namespace names that the term + applies to. The term is applied to the + union of the namespaces listed in this + field and the ones selected by namespaceSelector. + null or empty namespaces list and null + namespaceSelector means "this pod's + namespace". + items: + type: string + type: array + topologyKey: + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where co-located + is defined as running on a node whose + value of the label with key topologyKey + matches that of any node on which any + of the selected pods is running. Empty + topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + automountServiceAccountToken: + description: AutomountServiceAccountToken indicates + whether a service account token should be automatically + mounted. + type: boolean + containers: + description: List of containers belonging to the pod. + Containers cannot currently be added or removed. There + must be at least one container in a Pod. Cannot be + updated. + items: + description: A single application container that you + want to run within a pod. + properties: + args: + description: 'Arguments to the entrypoint. The + container image''s CMD is used if this is not + provided. Variable references $(VAR_NAME) are + expanded using the container''s environment. + If a variable cannot be resolved, the reference + in the input string will be unchanged. Double + $$ are reduced to a single $, which allows for + escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" + will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot + be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + command: + description: 'Entrypoint array. Not executed within + a shell. The container image''s ENTRYPOINT is + used if this is not provided. Variable references + $(VAR_NAME) are expanded using the container''s + environment. If a variable cannot be resolved, + the reference in the input string will be unchanged. + Double $$ are reduced to a single $, which allows + for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" + will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot + be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + env: + description: List of environment variables to + set in the container. Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) + are expanded using the previously defined + environment variables in the container + and any service environment variables. + If a variable cannot be resolved, the + reference in the input string will be + unchanged. Double $$ are reduced to a + single $, which allows for escaping the + $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" + will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, + regardless of whether the variable exists + or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used if value + is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. + apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the + ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: 'Selects a field of the + pod: supports metadata.name, metadata.namespace, + `metadata.labels['''']`, `metadata.annotations['''']`, + spec.nodeName, spec.serviceAccountName, + status.hostIP, status.podIP, status.podIPs.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: 'Selects a resource of + the container: only resources limits + and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required + for volumes, optional for env + vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. + apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the + Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + description: List of sources to populate environment + variables in the container. The keys defined + within a source must be a C_IDENTIFIER. All + invalid keys will be reported as an event when + the container is starting. When a key exists + in multiple sources, the value associated with + the last source will take precedence. Values + defined by an Env with a duplicate key will + take precedence. Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be + a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + image: + description: 'Container image name. More info: + https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level + config management to default or override container + images in workload controllers like Deployments + and StatefulSets.' + type: string + imagePullPolicy: + description: 'Image pull policy. One of Always, + Never, IfNotPresent. Defaults to Always if :latest + tag is specified, or IfNotPresent otherwise. + Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' + type: string + lifecycle: + description: Actions that the management system + should take in response to container lifecycle + events. Cannot be updated. + properties: + postStart: + description: 'PostStart is called immediately + after a container is created. If the handler + fails, the container is terminated and restarted + according to its restart policy. Other management + of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: Command is the command + line to execute inside the container, + the working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it + is not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to + explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: Host name to connect + to, defaults to the pod IP. You + probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: The header field + name. This will be canonicalized + upon output, so case-variant + names will be understood as + the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: Deprecated. TCPSocket is + NOT supported as a LifecycleHandler + and kept for the backward compatibility. + There are no validation of this field + and lifecycle hooks will fail in runtime + when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: 'PreStop is called immediately + before a container is terminated due to + an API request or management event such + as liveness/startup probe failure, preemption, + resource contention, etc. The handler is + not called if the container crashes or exits. + The Pod''s termination grace period countdown + begins before the PreStop hook is executed. + Regardless of the outcome of the handler, + the container will eventually terminate + within the Pod''s termination grace period + (unless delayed by finalizers). Other management + of the container blocks until the hook completes + or until the termination grace period is + reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: Command is the command + line to execute inside the container, + the working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it + is not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to + explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: Host name to connect + to, defaults to the pod IP. You + probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: The header field + name. This will be canonicalized + upon output, so case-variant + names will be understood as + the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: Deprecated. TCPSocket is + NOT supported as a LifecycleHandler + and kept for the backward compatibility. + There are no validation of this field + and lifecycle hooks will fail in runtime + when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: 'Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: Command is the command line + to execute inside the container, the + working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it is + not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to explicitly + call out to that shell. Exit status + of 0 is treated as live/healthy and + non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures + for the probe to be considered failed after + having succeeded. Defaults to 3. Minimum + value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the + service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default + behavior is defined by gRPC." + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, + defaults to the pod IP. You probably + want to set "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: The header field name. + This will be canonicalized upon + output, so case-variant names + will be understood as the same + header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the + container has started before liveness probes + are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform + the probe. Default to 10 seconds. Minimum + value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes + for the probe to be considered successful + after having failed. Defaults to 1. Must + be 1 for liveness and startup. Minimum value + is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds + the pod needs to terminate gracefully upon + probe failure. The grace period is the duration + in seconds after the processes running in + the pod are sent a termination signal and + the time when the processes are forcibly + halted with a kill signal. Set this value + longer than the expected cleanup time for + your process. If this value is nil, the + pod's terminationGracePeriodSeconds will + be used. Otherwise, this value overrides + the value provided by the pod spec. Value + must be non-negative integer. The value + zero indicates stop immediately via the + kill signal (no opportunity to shut down). + This is a beta field and requires enabling + ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which + the probe times out. Defaults to 1 second. + Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + name: + description: Name of the container specified as + a DNS_LABEL. Each container in a pod must have + a unique name (DNS_LABEL). Cannot be updated. + type: string + ports: + description: List of ports to expose from the + container. Not specifying a port here DOES NOT + prevent that port from being exposed. Any port + which is listening on the default "0.0.0.0" + address inside a container will be accessible + from the network. Modifying this array with + strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: Number of port to expose on + the pod's IP address. This must be a valid + port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: Number of port to expose on + the host. If specified, this must be a + valid port number, 0 < x < 65536. If HostNetwork + is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: If specified, this must be + an IANA_SVC_NAME and unique within the + pod. Each named port in a pod must have + a unique name. Name for the port that + can be referred to by services. + type: string + protocol: + default: TCP + description: Protocol for port. Must be + UDP, TCP, or SCTP. Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: 'Periodic probe of container service + readiness. Container will be removed from service + endpoints if the probe fails. Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: Command is the command line + to execute inside the container, the + working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it is + not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to explicitly + call out to that shell. Exit status + of 0 is treated as live/healthy and + non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures + for the probe to be considered failed after + having succeeded. Defaults to 3. Minimum + value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the + service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default + behavior is defined by gRPC." + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, + defaults to the pod IP. You probably + want to set "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: The header field name. + This will be canonicalized upon + output, so case-variant names + will be understood as the same + header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the + container has started before liveness probes + are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform + the probe. Default to 10 seconds. Minimum + value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes + for the probe to be considered successful + after having failed. Defaults to 1. Must + be 1 for liveness and startup. Minimum value + is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds + the pod needs to terminate gracefully upon + probe failure. The grace period is the duration + in seconds after the processes running in + the pod are sent a termination signal and + the time when the processes are forcibly + halted with a kill signal. Set this value + longer than the expected cleanup time for + your process. If this value is nil, the + pod's terminationGracePeriodSeconds will + be used. Otherwise, this value overrides + the value provided by the pod spec. Value + must be non-negative integer. The value + zero indicates stop immediately via the + kill signal (no opportunity to shut down). + This is a beta field and requires enabling + ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which + the probe times out. Defaults to 1 second. + Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: 'Name of the resource to which + this resource resize policy applies. Supported + values: cpu, memory.' + type: string + restartPolicy: + description: Restart policy to apply when + specified resource is resized. If not + specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: 'Compute Resources required by this + container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + properties: + claims: + description: "Claims lists the names of resources, + defined in spec.resourceClaims, that are + used by this container. \n This is an alpha + field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. + It can only be set for containers." + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name + of one entry in pod.spec.resourceClaims + of the Pod where this field is used. + It makes that resource available inside + a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum + amount of compute resources allowed. More + info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum + amount of compute resources required. If + Requests is omitted for a container, it + defaults to Limits if that is explicitly + specified, otherwise to an implementation-defined + value. Requests cannot exceed Limits. More + info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + securityContext: + description: 'SecurityContext defines the security + options the container should be run with. If + set, the fields of SecurityContext override + the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls + whether a process can gain more privileges + than its parent process. This bool directly + controls if the no_new_privs flag will be + set on the container process. AllowPrivilegeEscalation + is true always when the container is: 1) + run as Privileged 2) has CAP_SYS_ADMIN Note + that this field cannot be set when spec.os.name + is windows.' + type: boolean + capabilities: + description: The capabilities to add/drop + when running containers. Defaults to the + default set of capabilities granted by the + container runtime. Note that this field + cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + type: object + privileged: + description: Run container in privileged mode. + Processes in privileged containers are essentially + equivalent to root on the host. Defaults + to false. Note that this field cannot be + set when spec.os.name is windows. + type: boolean + procMount: + description: procMount denotes the type of + proc mount to use for the containers. The + default is DefaultProcMount which uses the + container runtime defaults for readonly + paths and masked paths. This requires the + ProcMountType feature flag to be enabled. + Note that this field cannot be set when + spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: Whether this container has a + read-only root filesystem. Default is false. + Note that this field cannot be set when + spec.os.name is windows. + type: boolean + runAsGroup: + description: The GID to run the entrypoint + of the container process. Uses runtime default + if unset. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. Note that this field cannot + be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container + must run as a non-root user. If true, the + Kubelet will validate the image at runtime + to ensure that it does not run as UID 0 + (root) and fail to start the container if + it does. If unset or false, no such validation + will be performed. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint + of the container process. Defaults to user + specified in image metadata if unspecified. + May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. Note that this field cannot + be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied + to the container. If unspecified, the container + runtime will allocate a random SELinux context + for each container. May also be set in + PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified + in SecurityContext takes precedence. Note + that this field cannot be set when spec.os.name + is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: The seccomp options to use by + this container. If seccomp options are provided + at both the pod & container level, the container + options override the pod options. Note that + this field cannot be set when spec.os.name + is windows. + properties: + localhostProfile: + description: localhostProfile indicates + a profile defined in a file on the node + should be used. The profile must be + preconfigured on the node to work. Must + be a descending path, relative to the + kubelet's configured seccomp profile + location. Must only be set if type is + "Localhost". + type: string + type: + description: "type indicates which kind + of seccomp profile will be applied. + Valid options are: \n Localhost - a + profile defined in a file on the node + should be used. RuntimeDefault - the + container runtime default profile should + be used. Unconfined - no profile should + be applied." + type: string + required: + - type + type: object + windowsOptions: + description: The Windows specific settings + applied to all containers. If unspecified, + the options from the PodSecurityContext + will be used. If set in both SecurityContext + and PodSecurityContext, the value specified + in SecurityContext takes precedence. Note + that this field cannot be set when spec.os.name + is linux. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where + the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential + spec named by the GMSACredentialSpecName + field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is + the name of the GMSA credential spec + to use. + type: string + hostProcess: + description: HostProcess determines if + a container should be run as a 'Host + Process' container. This field is alpha-level + and will only be honored by components + that enable the WindowsHostProcessContainers + feature flag. Setting this field without + the feature flag will result in errors + when validating the Pod. All of a Pod's + containers must have the same effective + HostProcess value (it is not allowed + to have a mix of HostProcess containers + and non-HostProcess containers). In + addition, if HostProcess is true then + HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: The UserName in Windows to + run the entrypoint of the container + process. Defaults to the user specified + in image metadata if unspecified. May + also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext + takes precedence. + type: string + type: object + type: object + startupProbe: + description: 'StartupProbe indicates that the + Pod has successfully initialized. If specified, + no other probes are executed until this completes + successfully. If this probe fails, the Pod will + be restarted, just as if the livenessProbe failed. + This can be used to provide different probe + parameters at the beginning of a Pod''s lifecycle, + when it might take a long time to load data + or warm a cache, than during steady-state operation. + This cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: Command is the command line + to execute inside the container, the + working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it is + not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to explicitly + call out to that shell. Exit status + of 0 is treated as live/healthy and + non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures + for the probe to be considered failed after + having succeeded. Defaults to 3. Minimum + value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the + service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default + behavior is defined by gRPC." + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, + defaults to the pod IP. You probably + want to set "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: The header field name. + This will be canonicalized upon + output, so case-variant names + will be understood as the same + header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the + container has started before liveness probes + are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform + the probe. Default to 10 seconds. Minimum + value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes + for the probe to be considered successful + after having failed. Defaults to 1. Must + be 1 for liveness and startup. Minimum value + is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds + the pod needs to terminate gracefully upon + probe failure. The grace period is the duration + in seconds after the processes running in + the pod are sent a termination signal and + the time when the processes are forcibly + halted with a kill signal. Set this value + longer than the expected cleanup time for + your process. If this value is nil, the + pod's terminationGracePeriodSeconds will + be used. Otherwise, this value overrides + the value provided by the pod spec. Value + must be non-negative integer. The value + zero indicates stop immediately via the + kill signal (no opportunity to shut down). + This is a beta field and requires enabling + ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which + the probe times out. Defaults to 1 second. + Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + stdin: + description: Whether this container should allocate + a buffer for stdin in the container runtime. + If this is not set, reads from stdin in the + container will always result in EOF. Default + is false. + type: boolean + stdinOnce: + description: Whether the container runtime should + close the stdin channel after it has been opened + by a single attach. When stdin is true the stdin + stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin + is opened on container start, is empty until + the first client attaches to stdin, and then + remains open and accepts data until the client + disconnects, at which time stdin is closed and + remains closed until the container is restarted. + If this flag is false, a container processes + that reads from stdin will never receive an + EOF. Default is false + type: boolean + terminationMessagePath: + description: 'Optional: Path at which the file + to which the container''s termination message + will be written is mounted into the container''s + filesystem. Message written is intended to be + brief final status, such as an assertion failure + message. Will be truncated by the node if greater + than 4096 bytes. The total message length across + all containers will be limited to 12kb. Defaults + to /dev/termination-log. Cannot be updated.' + type: string + terminationMessagePolicy: + description: Indicate how the termination message + should be populated. File will use the contents + of terminationMessagePath to populate the container + status message on both success and failure. + FallbackToLogsOnError will use the last chunk + of container log output if the termination message + file is empty and the container exited with + an error. The log output is limited to 2048 + bytes or 80 lines, whichever is smaller. Defaults + to File. Cannot be updated. + type: string + tty: + description: Whether this container should allocate + a TTY for itself, also requires 'stdin' to be + true. Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will + be mapped to. + type: string + name: + description: name must match the name of + a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + description: Pod volumes to mount into the container's + filesystem. Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: Path within the container at + which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: mountPropagation determines + how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is + used. This field is beta in 1.10. + type: string + name: + description: This must match the Name of + a Volume. + type: string + readOnly: + description: Mounted read-only if true, + read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + subPath: + description: Path within the volume from + which the container's volume should be + mounted. Defaults to "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume + from which the container's volume should + be mounted. Behaves similarly to SubPath + but environment variable references $(VAR_NAME) + are expanded using the container's environment. + Defaults to "" (volume's root). SubPathExpr + and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + description: Container's working directory. If + not specified, the container runtime's default + will be used, which might be configured in the + container image. Cannot be updated. + type: string + required: + - name + type: object + type: array + dnsConfig: + description: Specifies the DNS parameters of a pod. + Parameters specified here will be merged to the generated + DNS configuration based on DNSPolicy. + properties: + nameservers: + description: A list of DNS name server IP addresses. + This will be appended to the base nameservers + generated from DNSPolicy. Duplicated nameservers + will be removed. + items: + type: string + type: array + options: + description: A list of DNS resolver options. This + will be merged with the base options generated + from DNSPolicy. Duplicated entries will be removed. + Resolution options given in Options will override + those that appear in the base DNSPolicy. + items: + description: PodDNSConfigOption defines DNS resolver + options of a pod. + properties: + name: + description: Required. + type: string + value: + type: string + type: object + type: array + searches: + description: A list of DNS search domains for host-name + lookup. This will be appended to the base search + paths generated from DNSPolicy. Duplicated search + paths will be removed. + items: + type: string + type: array + type: object + dnsPolicy: + description: Set DNS policy for the pod. Defaults to + "ClusterFirst". Valid values are 'ClusterFirstWithHostNet', + 'ClusterFirst', 'Default' or 'None'. DNS parameters + given in DNSConfig will be merged with the policy + selected with DNSPolicy. To have DNS options set along + with hostNetwork, you have to specify DNS policy explicitly + to 'ClusterFirstWithHostNet'. + type: string + enableServiceLinks: + description: 'EnableServiceLinks indicates whether information + about services should be injected into pod''s environment + variables, matching the syntax of Docker links. Optional: + Defaults to true.' + type: boolean + ephemeralContainers: + description: List of ephemeral containers run in this + pod. Ephemeral containers may be run in an existing + pod to perform user-initiated actions such as debugging. + This list cannot be specified when creating a pod, + and it cannot be modified by updating the pod spec. + In order to add an ephemeral container to an existing + pod, use the pod's ephemeralcontainers subresource. + items: + description: "An EphemeralContainer is a temporary + container that you may add to an existing Pod for + user-initiated activities such as debugging. Ephemeral + containers have no resource or scheduling guarantees, + and they will not be restarted when they exit or + when a Pod is removed or restarted. The kubelet + may evict a Pod if an ephemeral container causes + the Pod to exceed its resource allocation. \n To + add an ephemeral container, use the ephemeralcontainers + subresource of an existing Pod. Ephemeral containers + may not be removed or restarted." + properties: + args: + description: 'Arguments to the entrypoint. The + image''s CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded + using the container''s environment. If a variable + cannot be resolved, the reference in the input + string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the + $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped + references will never be expanded, regardless + of whether the variable exists or not. Cannot + be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + command: + description: 'Entrypoint array. Not executed within + a shell. The image''s ENTRYPOINT is used if + this is not provided. Variable references $(VAR_NAME) + are expanded using the container''s environment. + If a variable cannot be resolved, the reference + in the input string will be unchanged. Double + $$ are reduced to a single $, which allows for + escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" + will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot + be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + env: + description: List of environment variables to + set in the container. Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) + are expanded using the previously defined + environment variables in the container + and any service environment variables. + If a variable cannot be resolved, the + reference in the input string will be + unchanged. Double $$ are reduced to a + single $, which allows for escaping the + $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" + will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, + regardless of whether the variable exists + or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used if value + is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. + apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the + ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: 'Selects a field of the + pod: supports metadata.name, metadata.namespace, + `metadata.labels['''']`, `metadata.annotations['''']`, + spec.nodeName, spec.serviceAccountName, + status.hostIP, status.podIP, status.podIPs.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: 'Selects a resource of + the container: only resources limits + and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required + for volumes, optional for env + vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. + apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the + Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + description: List of sources to populate environment + variables in the container. The keys defined + within a source must be a C_IDENTIFIER. All + invalid keys will be reported as an event when + the container is starting. When a key exists + in multiple sources, the value associated with + the last source will take precedence. Values + defined by an Env with a duplicate key will + take precedence. Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be + a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + image: + description: 'Container image name. More info: + https://kubernetes.io/docs/concepts/containers/images' + type: string + imagePullPolicy: + description: 'Image pull policy. One of Always, + Never, IfNotPresent. Defaults to Always if :latest + tag is specified, or IfNotPresent otherwise. + Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' + type: string + lifecycle: + description: Lifecycle is not allowed for ephemeral + containers. + properties: + postStart: + description: 'PostStart is called immediately + after a container is created. If the handler + fails, the container is terminated and restarted + according to its restart policy. Other management + of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: Command is the command + line to execute inside the container, + the working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it + is not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to + explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: Host name to connect + to, defaults to the pod IP. You + probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: The header field + name. This will be canonicalized + upon output, so case-variant + names will be understood as + the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: Deprecated. TCPSocket is + NOT supported as a LifecycleHandler + and kept for the backward compatibility. + There are no validation of this field + and lifecycle hooks will fail in runtime + when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: 'PreStop is called immediately + before a container is terminated due to + an API request or management event such + as liveness/startup probe failure, preemption, + resource contention, etc. The handler is + not called if the container crashes or exits. + The Pod''s termination grace period countdown + begins before the PreStop hook is executed. + Regardless of the outcome of the handler, + the container will eventually terminate + within the Pod''s termination grace period + (unless delayed by finalizers). Other management + of the container blocks until the hook completes + or until the termination grace period is + reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: Command is the command + line to execute inside the container, + the working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it + is not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to + explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: Host name to connect + to, defaults to the pod IP. You + probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: The header field + name. This will be canonicalized + upon output, so case-variant + names will be understood as + the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: Deprecated. TCPSocket is + NOT supported as a LifecycleHandler + and kept for the backward compatibility. + There are no validation of this field + and lifecycle hooks will fail in runtime + when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: Command is the command line + to execute inside the container, the + working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it is + not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to explicitly + call out to that shell. Exit status + of 0 is treated as live/healthy and + non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures + for the probe to be considered failed after + having succeeded. Defaults to 3. Minimum + value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the + service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default + behavior is defined by gRPC." + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, + defaults to the pod IP. You probably + want to set "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: The header field name. + This will be canonicalized upon + output, so case-variant names + will be understood as the same + header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the + container has started before liveness probes + are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform + the probe. Default to 10 seconds. Minimum + value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes + for the probe to be considered successful + after having failed. Defaults to 1. Must + be 1 for liveness and startup. Minimum value + is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds + the pod needs to terminate gracefully upon + probe failure. The grace period is the duration + in seconds after the processes running in + the pod are sent a termination signal and + the time when the processes are forcibly + halted with a kill signal. Set this value + longer than the expected cleanup time for + your process. If this value is nil, the + pod's terminationGracePeriodSeconds will + be used. Otherwise, this value overrides + the value provided by the pod spec. Value + must be non-negative integer. The value + zero indicates stop immediately via the + kill signal (no opportunity to shut down). + This is a beta field and requires enabling + ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which + the probe times out. Defaults to 1 second. + Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + name: + description: Name of the ephemeral container specified + as a DNS_LABEL. This name must be unique among + all containers, init containers and ephemeral + containers. + type: string + ports: + description: Ports are not allowed for ephemeral + containers. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: Number of port to expose on + the pod's IP address. This must be a valid + port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: Number of port to expose on + the host. If specified, this must be a + valid port number, 0 < x < 65536. If HostNetwork + is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: If specified, this must be + an IANA_SVC_NAME and unique within the + pod. Each named port in a pod must have + a unique name. Name for the port that + can be referred to by services. + type: string + protocol: + default: TCP + description: Protocol for port. Must be + UDP, TCP, or SCTP. Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: Command is the command line + to execute inside the container, the + working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it is + not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to explicitly + call out to that shell. Exit status + of 0 is treated as live/healthy and + non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures + for the probe to be considered failed after + having succeeded. Defaults to 3. Minimum + value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the + service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default + behavior is defined by gRPC." + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, + defaults to the pod IP. You probably + want to set "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: The header field name. + This will be canonicalized upon + output, so case-variant names + will be understood as the same + header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the + container has started before liveness probes + are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform + the probe. Default to 10 seconds. Minimum + value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes + for the probe to be considered successful + after having failed. Defaults to 1. Must + be 1 for liveness and startup. Minimum value + is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds + the pod needs to terminate gracefully upon + probe failure. The grace period is the duration + in seconds after the processes running in + the pod are sent a termination signal and + the time when the processes are forcibly + halted with a kill signal. Set this value + longer than the expected cleanup time for + your process. If this value is nil, the + pod's terminationGracePeriodSeconds will + be used. Otherwise, this value overrides + the value provided by the pod spec. Value + must be non-negative integer. The value + zero indicates stop immediately via the + kill signal (no opportunity to shut down). + This is a beta field and requires enabling + ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which + the probe times out. Defaults to 1 second. + Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: 'Name of the resource to which + this resource resize policy applies. Supported + values: cpu, memory.' + type: string + restartPolicy: + description: Restart policy to apply when + specified resource is resized. If not + specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: Resources are not allowed for ephemeral + containers. Ephemeral containers use spare resources + already allocated to the pod. + properties: + claims: + description: "Claims lists the names of resources, + defined in spec.resourceClaims, that are + used by this container. \n This is an alpha + field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. + It can only be set for containers." + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name + of one entry in pod.spec.resourceClaims + of the Pod where this field is used. + It makes that resource available inside + a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum + amount of compute resources allowed. More + info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum + amount of compute resources required. If + Requests is omitted for a container, it + defaults to Limits if that is explicitly + specified, otherwise to an implementation-defined + value. Requests cannot exceed Limits. More + info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + securityContext: + description: 'Optional: SecurityContext defines + the security options the ephemeral container + should be run with. If set, the fields of SecurityContext + override the equivalent fields of PodSecurityContext.' + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls + whether a process can gain more privileges + than its parent process. This bool directly + controls if the no_new_privs flag will be + set on the container process. AllowPrivilegeEscalation + is true always when the container is: 1) + run as Privileged 2) has CAP_SYS_ADMIN Note + that this field cannot be set when spec.os.name + is windows.' + type: boolean + capabilities: + description: The capabilities to add/drop + when running containers. Defaults to the + default set of capabilities granted by the + container runtime. Note that this field + cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + type: object + privileged: + description: Run container in privileged mode. + Processes in privileged containers are essentially + equivalent to root on the host. Defaults + to false. Note that this field cannot be + set when spec.os.name is windows. + type: boolean + procMount: + description: procMount denotes the type of + proc mount to use for the containers. The + default is DefaultProcMount which uses the + container runtime defaults for readonly + paths and masked paths. This requires the + ProcMountType feature flag to be enabled. + Note that this field cannot be set when + spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: Whether this container has a + read-only root filesystem. Default is false. + Note that this field cannot be set when + spec.os.name is windows. + type: boolean + runAsGroup: + description: The GID to run the entrypoint + of the container process. Uses runtime default + if unset. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. Note that this field cannot + be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container + must run as a non-root user. If true, the + Kubelet will validate the image at runtime + to ensure that it does not run as UID 0 + (root) and fail to start the container if + it does. If unset or false, no such validation + will be performed. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint + of the container process. Defaults to user + specified in image metadata if unspecified. + May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. Note that this field cannot + be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied + to the container. If unspecified, the container + runtime will allocate a random SELinux context + for each container. May also be set in + PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified + in SecurityContext takes precedence. Note + that this field cannot be set when spec.os.name + is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: The seccomp options to use by + this container. If seccomp options are provided + at both the pod & container level, the container + options override the pod options. Note that + this field cannot be set when spec.os.name + is windows. + properties: + localhostProfile: + description: localhostProfile indicates + a profile defined in a file on the node + should be used. The profile must be + preconfigured on the node to work. Must + be a descending path, relative to the + kubelet's configured seccomp profile + location. Must only be set if type is + "Localhost". + type: string + type: + description: "type indicates which kind + of seccomp profile will be applied. + Valid options are: \n Localhost - a + profile defined in a file on the node + should be used. RuntimeDefault - the + container runtime default profile should + be used. Unconfined - no profile should + be applied." + type: string + required: + - type + type: object + windowsOptions: + description: The Windows specific settings + applied to all containers. If unspecified, + the options from the PodSecurityContext + will be used. If set in both SecurityContext + and PodSecurityContext, the value specified + in SecurityContext takes precedence. Note + that this field cannot be set when spec.os.name + is linux. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where + the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential + spec named by the GMSACredentialSpecName + field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is + the name of the GMSA credential spec + to use. + type: string + hostProcess: + description: HostProcess determines if + a container should be run as a 'Host + Process' container. This field is alpha-level + and will only be honored by components + that enable the WindowsHostProcessContainers + feature flag. Setting this field without + the feature flag will result in errors + when validating the Pod. All of a Pod's + containers must have the same effective + HostProcess value (it is not allowed + to have a mix of HostProcess containers + and non-HostProcess containers). In + addition, if HostProcess is true then + HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: The UserName in Windows to + run the entrypoint of the container + process. Defaults to the user specified + in image metadata if unspecified. May + also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext + takes precedence. + type: string + type: object + type: object + startupProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: Command is the command line + to execute inside the container, the + working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it is + not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to explicitly + call out to that shell. Exit status + of 0 is treated as live/healthy and + non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures + for the probe to be considered failed after + having succeeded. Defaults to 3. Minimum + value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the + service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default + behavior is defined by gRPC." + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, + defaults to the pod IP. You probably + want to set "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: The header field name. + This will be canonicalized upon + output, so case-variant names + will be understood as the same + header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the + container has started before liveness probes + are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform + the probe. Default to 10 seconds. Minimum + value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes + for the probe to be considered successful + after having failed. Defaults to 1. Must + be 1 for liveness and startup. Minimum value + is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds + the pod needs to terminate gracefully upon + probe failure. The grace period is the duration + in seconds after the processes running in + the pod are sent a termination signal and + the time when the processes are forcibly + halted with a kill signal. Set this value + longer than the expected cleanup time for + your process. If this value is nil, the + pod's terminationGracePeriodSeconds will + be used. Otherwise, this value overrides + the value provided by the pod spec. Value + must be non-negative integer. The value + zero indicates stop immediately via the + kill signal (no opportunity to shut down). + This is a beta field and requires enabling + ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which + the probe times out. Defaults to 1 second. + Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + stdin: + description: Whether this container should allocate + a buffer for stdin in the container runtime. + If this is not set, reads from stdin in the + container will always result in EOF. Default + is false. + type: boolean + stdinOnce: + description: Whether the container runtime should + close the stdin channel after it has been opened + by a single attach. When stdin is true the stdin + stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin + is opened on container start, is empty until + the first client attaches to stdin, and then + remains open and accepts data until the client + disconnects, at which time stdin is closed and + remains closed until the container is restarted. + If this flag is false, a container processes + that reads from stdin will never receive an + EOF. Default is false + type: boolean + targetContainerName: + description: "If set, the name of the container + from PodSpec that this ephemeral container targets. + The ephemeral container will be run in the namespaces + (IPC, PID, etc) of this container. If not set + then the ephemeral container uses the namespaces + configured in the Pod spec. \n The container + runtime must implement support for this feature. + If the runtime does not support namespace targeting + then the result of setting this field is undefined." + type: string + terminationMessagePath: + description: 'Optional: Path at which the file + to which the container''s termination message + will be written is mounted into the container''s + filesystem. Message written is intended to be + brief final status, such as an assertion failure + message. Will be truncated by the node if greater + than 4096 bytes. The total message length across + all containers will be limited to 12kb. Defaults + to /dev/termination-log. Cannot be updated.' + type: string + terminationMessagePolicy: + description: Indicate how the termination message + should be populated. File will use the contents + of terminationMessagePath to populate the container + status message on both success and failure. + FallbackToLogsOnError will use the last chunk + of container log output if the termination message + file is empty and the container exited with + an error. The log output is limited to 2048 + bytes or 80 lines, whichever is smaller. Defaults + to File. Cannot be updated. + type: string + tty: + description: Whether this container should allocate + a TTY for itself, also requires 'stdin' to be + true. Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will + be mapped to. + type: string + name: + description: name must match the name of + a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + description: Pod volumes to mount into the container's + filesystem. Subpath mounts are not allowed for + ephemeral containers. Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: Path within the container at + which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: mountPropagation determines + how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is + used. This field is beta in 1.10. + type: string + name: + description: This must match the Name of + a Volume. + type: string + readOnly: + description: Mounted read-only if true, + read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + subPath: + description: Path within the volume from + which the container's volume should be + mounted. Defaults to "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume + from which the container's volume should + be mounted. Behaves similarly to SubPath + but environment variable references $(VAR_NAME) + are expanded using the container's environment. + Defaults to "" (volume's root). SubPathExpr + and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + description: Container's working directory. If + not specified, the container runtime's default + will be used, which might be configured in the + container image. Cannot be updated. + type: string + required: + - name + type: object + type: array + hostAliases: + description: HostAliases is an optional list of hosts + and IPs that will be injected into the pod's hosts + file if specified. This is only valid for non-hostNetwork + pods. + items: + description: HostAlias holds the mapping between IP + and hostnames that will be injected as an entry + in the pod's hosts file. + properties: + hostnames: + description: Hostnames for the above IP address. + items: + type: string + type: array + ip: + description: IP address of the host file entry. + type: string + type: object + type: array + hostIPC: + description: 'Use the host''s ipc namespace. Optional: + Default to false.' + type: boolean + hostNetwork: + description: Host networking requested for this pod. + Use the host's network namespace. If this option is + set, the ports that will be used must be specified. + Default to false. + type: boolean + hostPID: + description: 'Use the host''s pid namespace. Optional: + Default to false.' + type: boolean + hostUsers: + description: 'Use the host''s user namespace. Optional: + Default to true. If set to true or not present, the + pod will be run in the host user namespace, useful + for when the pod needs a feature only available to + the host user namespace, such as loading a kernel + module with CAP_SYS_MODULE. When set to false, a new + userns is created for the pod. Setting false is useful + for mitigating container breakout vulnerabilities + even allowing users to run their containers as root + without actually having root privileges on the host. + This field is alpha-level and is only honored by servers + that enable the UserNamespacesSupport feature.' + type: boolean + hostname: + description: Specifies the hostname of the Pod If not + specified, the pod's hostname will be set to a system-defined + value. + type: string + imagePullSecrets: + description: 'ImagePullSecrets is an optional list of + references to secrets in the same namespace to use + for pulling any of the images used by this PodSpec. + If specified, these secrets will be passed to individual + puller implementations for them to use. More info: + https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod' + items: + description: LocalObjectReference contains enough + information to let you locate the referenced object + inside the same namespace. + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + x-kubernetes-map-type: atomic + type: array + initContainers: + description: 'List of initialization containers belonging + to the pod. Init containers are executed in order + prior to containers being started. If any init container + fails, the pod is considered to have failed and is + handled according to its restartPolicy. The name for + an init container or normal container must be unique + among all containers. Init containers may not have + Lifecycle actions, Readiness probes, Liveness probes, + or Startup probes. The resourceRequirements of an + init container are taken into account during scheduling + by finding the highest request/limit for each resource + type, and then using the max of of that value or the + sum of the normal containers. Limits are applied to + init containers in a similar fashion. Init containers + cannot currently be added or removed. Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/' + items: + description: A single application container that you + want to run within a pod. + properties: + args: + description: 'Arguments to the entrypoint. The + container image''s CMD is used if this is not + provided. Variable references $(VAR_NAME) are + expanded using the container''s environment. + If a variable cannot be resolved, the reference + in the input string will be unchanged. Double + $$ are reduced to a single $, which allows for + escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" + will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot + be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + command: + description: 'Entrypoint array. Not executed within + a shell. The container image''s ENTRYPOINT is + used if this is not provided. Variable references + $(VAR_NAME) are expanded using the container''s + environment. If a variable cannot be resolved, + the reference in the input string will be unchanged. + Double $$ are reduced to a single $, which allows + for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" + will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot + be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + env: + description: List of environment variables to + set in the container. Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) + are expanded using the previously defined + environment variables in the container + and any service environment variables. + If a variable cannot be resolved, the + reference in the input string will be + unchanged. Double $$ are reduced to a + single $, which allows for escaping the + $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" + will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, + regardless of whether the variable exists + or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used if value + is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. + apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the + ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: 'Selects a field of the + pod: supports metadata.name, metadata.namespace, + `metadata.labels['''']`, `metadata.annotations['''']`, + spec.nodeName, spec.serviceAccountName, + status.hostIP, status.podIP, status.podIPs.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: 'Selects a resource of + the container: only resources limits + and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required + for volumes, optional for env + vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. + apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the + Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + description: List of sources to populate environment + variables in the container. The keys defined + within a source must be a C_IDENTIFIER. All + invalid keys will be reported as an event when + the container is starting. When a key exists + in multiple sources, the value associated with + the last source will take precedence. Values + defined by an Env with a duplicate key will + take precedence. Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be + a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + image: + description: 'Container image name. More info: + https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level + config management to default or override container + images in workload controllers like Deployments + and StatefulSets.' + type: string + imagePullPolicy: + description: 'Image pull policy. One of Always, + Never, IfNotPresent. Defaults to Always if :latest + tag is specified, or IfNotPresent otherwise. + Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' + type: string + lifecycle: + description: Actions that the management system + should take in response to container lifecycle + events. Cannot be updated. + properties: + postStart: + description: 'PostStart is called immediately + after a container is created. If the handler + fails, the container is terminated and restarted + according to its restart policy. Other management + of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: Command is the command + line to execute inside the container, + the working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it + is not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to + explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: Host name to connect + to, defaults to the pod IP. You + probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: The header field + name. This will be canonicalized + upon output, so case-variant + names will be understood as + the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: Deprecated. TCPSocket is + NOT supported as a LifecycleHandler + and kept for the backward compatibility. + There are no validation of this field + and lifecycle hooks will fail in runtime + when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: 'PreStop is called immediately + before a container is terminated due to + an API request or management event such + as liveness/startup probe failure, preemption, + resource contention, etc. The handler is + not called if the container crashes or exits. + The Pod''s termination grace period countdown + begins before the PreStop hook is executed. + Regardless of the outcome of the handler, + the container will eventually terminate + within the Pod''s termination grace period + (unless delayed by finalizers). Other management + of the container blocks until the hook completes + or until the termination grace period is + reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: Command is the command + line to execute inside the container, + the working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it + is not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to + explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: Host name to connect + to, defaults to the pod IP. You + probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: The header field + name. This will be canonicalized + upon output, so case-variant + names will be understood as + the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: Deprecated. TCPSocket is + NOT supported as a LifecycleHandler + and kept for the backward compatibility. + There are no validation of this field + and lifecycle hooks will fail in runtime + when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: 'Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: Command is the command line + to execute inside the container, the + working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it is + not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to explicitly + call out to that shell. Exit status + of 0 is treated as live/healthy and + non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures + for the probe to be considered failed after + having succeeded. Defaults to 3. Minimum + value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the + service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default + behavior is defined by gRPC." + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, + defaults to the pod IP. You probably + want to set "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: The header field name. + This will be canonicalized upon + output, so case-variant names + will be understood as the same + header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the + container has started before liveness probes + are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform + the probe. Default to 10 seconds. Minimum + value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes + for the probe to be considered successful + after having failed. Defaults to 1. Must + be 1 for liveness and startup. Minimum value + is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds + the pod needs to terminate gracefully upon + probe failure. The grace period is the duration + in seconds after the processes running in + the pod are sent a termination signal and + the time when the processes are forcibly + halted with a kill signal. Set this value + longer than the expected cleanup time for + your process. If this value is nil, the + pod's terminationGracePeriodSeconds will + be used. Otherwise, this value overrides + the value provided by the pod spec. Value + must be non-negative integer. The value + zero indicates stop immediately via the + kill signal (no opportunity to shut down). + This is a beta field and requires enabling + ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which + the probe times out. Defaults to 1 second. + Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + name: + description: Name of the container specified as + a DNS_LABEL. Each container in a pod must have + a unique name (DNS_LABEL). Cannot be updated. + type: string + ports: + description: List of ports to expose from the + container. Not specifying a port here DOES NOT + prevent that port from being exposed. Any port + which is listening on the default "0.0.0.0" + address inside a container will be accessible + from the network. Modifying this array with + strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: Number of port to expose on + the pod's IP address. This must be a valid + port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: Number of port to expose on + the host. If specified, this must be a + valid port number, 0 < x < 65536. If HostNetwork + is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: If specified, this must be + an IANA_SVC_NAME and unique within the + pod. Each named port in a pod must have + a unique name. Name for the port that + can be referred to by services. + type: string + protocol: + default: TCP + description: Protocol for port. Must be + UDP, TCP, or SCTP. Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: 'Periodic probe of container service + readiness. Container will be removed from service + endpoints if the probe fails. Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: Command is the command line + to execute inside the container, the + working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it is + not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to explicitly + call out to that shell. Exit status + of 0 is treated as live/healthy and + non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures + for the probe to be considered failed after + having succeeded. Defaults to 3. Minimum + value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the + service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default + behavior is defined by gRPC." + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, + defaults to the pod IP. You probably + want to set "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: The header field name. + This will be canonicalized upon + output, so case-variant names + will be understood as the same + header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the + container has started before liveness probes + are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform + the probe. Default to 10 seconds. Minimum + value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes + for the probe to be considered successful + after having failed. Defaults to 1. Must + be 1 for liveness and startup. Minimum value + is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds + the pod needs to terminate gracefully upon + probe failure. The grace period is the duration + in seconds after the processes running in + the pod are sent a termination signal and + the time when the processes are forcibly + halted with a kill signal. Set this value + longer than the expected cleanup time for + your process. If this value is nil, the + pod's terminationGracePeriodSeconds will + be used. Otherwise, this value overrides + the value provided by the pod spec. Value + must be non-negative integer. The value + zero indicates stop immediately via the + kill signal (no opportunity to shut down). + This is a beta field and requires enabling + ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which + the probe times out. Defaults to 1 second. + Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: 'Name of the resource to which + this resource resize policy applies. Supported + values: cpu, memory.' + type: string + restartPolicy: + description: Restart policy to apply when + specified resource is resized. If not + specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: 'Compute Resources required by this + container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + properties: + claims: + description: "Claims lists the names of resources, + defined in spec.resourceClaims, that are + used by this container. \n This is an alpha + field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. + It can only be set for containers." + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name + of one entry in pod.spec.resourceClaims + of the Pod where this field is used. + It makes that resource available inside + a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum + amount of compute resources allowed. More + info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum + amount of compute resources required. If + Requests is omitted for a container, it + defaults to Limits if that is explicitly + specified, otherwise to an implementation-defined + value. Requests cannot exceed Limits. More + info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + securityContext: + description: 'SecurityContext defines the security + options the container should be run with. If + set, the fields of SecurityContext override + the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls + whether a process can gain more privileges + than its parent process. This bool directly + controls if the no_new_privs flag will be + set on the container process. AllowPrivilegeEscalation + is true always when the container is: 1) + run as Privileged 2) has CAP_SYS_ADMIN Note + that this field cannot be set when spec.os.name + is windows.' + type: boolean + capabilities: + description: The capabilities to add/drop + when running containers. Defaults to the + default set of capabilities granted by the + container runtime. Note that this field + cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + type: object + privileged: + description: Run container in privileged mode. + Processes in privileged containers are essentially + equivalent to root on the host. Defaults + to false. Note that this field cannot be + set when spec.os.name is windows. + type: boolean + procMount: + description: procMount denotes the type of + proc mount to use for the containers. The + default is DefaultProcMount which uses the + container runtime defaults for readonly + paths and masked paths. This requires the + ProcMountType feature flag to be enabled. + Note that this field cannot be set when + spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: Whether this container has a + read-only root filesystem. Default is false. + Note that this field cannot be set when + spec.os.name is windows. + type: boolean + runAsGroup: + description: The GID to run the entrypoint + of the container process. Uses runtime default + if unset. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. Note that this field cannot + be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container + must run as a non-root user. If true, the + Kubelet will validate the image at runtime + to ensure that it does not run as UID 0 + (root) and fail to start the container if + it does. If unset or false, no such validation + will be performed. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint + of the container process. Defaults to user + specified in image metadata if unspecified. + May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. Note that this field cannot + be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied + to the container. If unspecified, the container + runtime will allocate a random SELinux context + for each container. May also be set in + PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified + in SecurityContext takes precedence. Note + that this field cannot be set when spec.os.name + is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: The seccomp options to use by + this container. If seccomp options are provided + at both the pod & container level, the container + options override the pod options. Note that + this field cannot be set when spec.os.name + is windows. + properties: + localhostProfile: + description: localhostProfile indicates + a profile defined in a file on the node + should be used. The profile must be + preconfigured on the node to work. Must + be a descending path, relative to the + kubelet's configured seccomp profile + location. Must only be set if type is + "Localhost". + type: string + type: + description: "type indicates which kind + of seccomp profile will be applied. + Valid options are: \n Localhost - a + profile defined in a file on the node + should be used. RuntimeDefault - the + container runtime default profile should + be used. Unconfined - no profile should + be applied." + type: string + required: + - type + type: object + windowsOptions: + description: The Windows specific settings + applied to all containers. If unspecified, + the options from the PodSecurityContext + will be used. If set in both SecurityContext + and PodSecurityContext, the value specified + in SecurityContext takes precedence. Note + that this field cannot be set when spec.os.name + is linux. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where + the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential + spec named by the GMSACredentialSpecName + field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is + the name of the GMSA credential spec + to use. + type: string + hostProcess: + description: HostProcess determines if + a container should be run as a 'Host + Process' container. This field is alpha-level + and will only be honored by components + that enable the WindowsHostProcessContainers + feature flag. Setting this field without + the feature flag will result in errors + when validating the Pod. All of a Pod's + containers must have the same effective + HostProcess value (it is not allowed + to have a mix of HostProcess containers + and non-HostProcess containers). In + addition, if HostProcess is true then + HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: The UserName in Windows to + run the entrypoint of the container + process. Defaults to the user specified + in image metadata if unspecified. May + also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext + takes precedence. + type: string + type: object + type: object + startupProbe: + description: 'StartupProbe indicates that the + Pod has successfully initialized. If specified, + no other probes are executed until this completes + successfully. If this probe fails, the Pod will + be restarted, just as if the livenessProbe failed. + This can be used to provide different probe + parameters at the beginning of a Pod''s lifecycle, + when it might take a long time to load data + or warm a cache, than during steady-state operation. + This cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: Command is the command line + to execute inside the container, the + working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it is + not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to explicitly + call out to that shell. Exit status + of 0 is treated as live/healthy and + non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures + for the probe to be considered failed after + having succeeded. Defaults to 3. Minimum + value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the + service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default + behavior is defined by gRPC." + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, + defaults to the pod IP. You probably + want to set "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: The header field name. + This will be canonicalized upon + output, so case-variant names + will be understood as the same + header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the + container has started before liveness probes + are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform + the probe. Default to 10 seconds. Minimum + value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes + for the probe to be considered successful + after having failed. Defaults to 1. Must + be 1 for liveness and startup. Minimum value + is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds + the pod needs to terminate gracefully upon + probe failure. The grace period is the duration + in seconds after the processes running in + the pod are sent a termination signal and + the time when the processes are forcibly + halted with a kill signal. Set this value + longer than the expected cleanup time for + your process. If this value is nil, the + pod's terminationGracePeriodSeconds will + be used. Otherwise, this value overrides + the value provided by the pod spec. Value + must be non-negative integer. The value + zero indicates stop immediately via the + kill signal (no opportunity to shut down). + This is a beta field and requires enabling + ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which + the probe times out. Defaults to 1 second. + Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + stdin: + description: Whether this container should allocate + a buffer for stdin in the container runtime. + If this is not set, reads from stdin in the + container will always result in EOF. Default + is false. + type: boolean + stdinOnce: + description: Whether the container runtime should + close the stdin channel after it has been opened + by a single attach. When stdin is true the stdin + stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin + is opened on container start, is empty until + the first client attaches to stdin, and then + remains open and accepts data until the client + disconnects, at which time stdin is closed and + remains closed until the container is restarted. + If this flag is false, a container processes + that reads from stdin will never receive an + EOF. Default is false + type: boolean + terminationMessagePath: + description: 'Optional: Path at which the file + to which the container''s termination message + will be written is mounted into the container''s + filesystem. Message written is intended to be + brief final status, such as an assertion failure + message. Will be truncated by the node if greater + than 4096 bytes. The total message length across + all containers will be limited to 12kb. Defaults + to /dev/termination-log. Cannot be updated.' + type: string + terminationMessagePolicy: + description: Indicate how the termination message + should be populated. File will use the contents + of terminationMessagePath to populate the container + status message on both success and failure. + FallbackToLogsOnError will use the last chunk + of container log output if the termination message + file is empty and the container exited with + an error. The log output is limited to 2048 + bytes or 80 lines, whichever is smaller. Defaults + to File. Cannot be updated. + type: string + tty: + description: Whether this container should allocate + a TTY for itself, also requires 'stdin' to be + true. Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will + be mapped to. + type: string + name: + description: name must match the name of + a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + description: Pod volumes to mount into the container's + filesystem. Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: Path within the container at + which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: mountPropagation determines + how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is + used. This field is beta in 1.10. + type: string + name: + description: This must match the Name of + a Volume. + type: string + readOnly: + description: Mounted read-only if true, + read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + subPath: + description: Path within the volume from + which the container's volume should be + mounted. Defaults to "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume + from which the container's volume should + be mounted. Behaves similarly to SubPath + but environment variable references $(VAR_NAME) + are expanded using the container's environment. + Defaults to "" (volume's root). SubPathExpr + and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + description: Container's working directory. If + not specified, the container runtime's default + will be used, which might be configured in the + container image. Cannot be updated. + type: string + required: + - name + type: object + type: array + nodeName: + description: NodeName is a request to schedule this + pod onto a specific node. If it is non-empty, the + scheduler simply schedules this pod onto that node, + assuming that it fits resource requirements. + type: string + nodeSelector: + additionalProperties: + type: string + description: 'NodeSelector is a selector which must + be true for the pod to fit on a node. Selector which + must match a node''s labels for the pod to be scheduled + on that node. More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/' + type: object + x-kubernetes-map-type: atomic + os: + description: "Specifies the OS of the containers in + the pod. Some pod and container fields are restricted + if this is set. \n If the OS field is set to linux, + the following fields must be unset: -securityContext.windowsOptions + \n If the OS field is set to windows, following fields + must be unset: - spec.hostPID - spec.hostIPC - spec.hostUsers + - spec.securityContext.seLinuxOptions - spec.securityContext.seccompProfile + - spec.securityContext.fsGroup - spec.securityContext.fsGroupChangePolicy + - spec.securityContext.sysctls - spec.shareProcessNamespace + - spec.securityContext.runAsUser - spec.securityContext.runAsGroup + - spec.securityContext.supplementalGroups - spec.containers[*].securityContext.seLinuxOptions + - spec.containers[*].securityContext.seccompProfile + - spec.containers[*].securityContext.capabilities + - spec.containers[*].securityContext.readOnlyRootFilesystem + - spec.containers[*].securityContext.privileged - + spec.containers[*].securityContext.allowPrivilegeEscalation + - spec.containers[*].securityContext.procMount - spec.containers[*].securityContext.runAsUser + - spec.containers[*].securityContext.runAsGroup" + properties: + name: + description: 'Name is the name of the operating + system. The currently supported values are linux + and windows. Additional value may be defined in + future and can be one of: https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration + Clients should expect to handle additional values + and treat unrecognized values in this field as + os: null' + type: string + required: + - name + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Overhead represents the resource overhead + associated with running a pod for a given RuntimeClass. + This field will be autopopulated at admission time + by the RuntimeClass admission controller. If the RuntimeClass + admission controller is enabled, overhead must not + be set in Pod create requests. The RuntimeClass admission + controller will reject Pod create requests which have + the overhead already set. If RuntimeClass is configured + and selected in the PodSpec, Overhead will be set + to the value defined in the corresponding RuntimeClass, + otherwise it will remain unset and treated as zero. + More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md' + type: object + preemptionPolicy: + description: PreemptionPolicy is the Policy for preempting + pods with lower priority. One of Never, PreemptLowerPriority. + Defaults to PreemptLowerPriority if unset. + type: string + priority: + description: The priority value. Various system components + use this field to find the priority of the pod. When + Priority Admission Controller is enabled, it prevents + users from setting this field. The admission controller + populates this field from PriorityClassName. The higher + the value, the higher the priority. + format: int32 + type: integer + priorityClassName: + description: If specified, indicates the pod's priority. + "system-node-critical" and "system-cluster-critical" + are two special keywords which indicate the highest + priorities with the former being the highest priority. + Any other name must be defined by creating a PriorityClass + object with that name. If not specified, the pod priority + will be default or zero if there is no default. + type: string + readinessGates: + description: 'If specified, all readiness gates will + be evaluated for pod readiness. A pod is ready when + all its containers are ready AND all conditions specified + in the readiness gates have status equal to "True" + More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates' + items: + description: PodReadinessGate contains the reference + to a pod condition + properties: + conditionType: + description: ConditionType refers to a condition + in the pod's condition list with matching type. + type: string + required: + - conditionType + type: object + type: array + resourceClaims: + description: "ResourceClaims defines which ResourceClaims + must be allocated and reserved before the Pod is allowed + to start. The resources will be made available to + those containers which consume them by name. \n This + is an alpha field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable." + items: + description: PodResourceClaim references exactly one + ResourceClaim through a ClaimSource. It adds a name + to it that uniquely identifies the ResourceClaim + inside the Pod. Containers that need access to the + ResourceClaim reference it with this name. + properties: + name: + description: Name uniquely identifies this resource + claim inside the pod. This must be a DNS_LABEL. + type: string + source: + description: Source describes where to find the + ResourceClaim. + properties: + resourceClaimName: + description: ResourceClaimName is the name + of a ResourceClaim object in the same namespace + as this pod. + type: string + resourceClaimTemplateName: + description: "ResourceClaimTemplateName is + the name of a ResourceClaimTemplate object + in the same namespace as this pod. \n The + template will be used to create a new ResourceClaim, + which will be bound to this pod. When this + pod is deleted, the ResourceClaim will also + be deleted. The name of the ResourceClaim + will be -, where + is the PodResourceClaim.Name. + Pod validation will reject the pod if the + concatenated name is not valid for a ResourceClaim + (e.g. too long). \n An existing ResourceClaim + with that name that is not owned by the + pod will not be used for the pod to avoid + using an unrelated resource by mistake. + Scheduling and pod startup are then blocked + until the unrelated ResourceClaim is removed. + \n This field is immutable and no changes + will be made to the corresponding ResourceClaim + by the control plane after creating the + ResourceClaim." + type: string + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + restartPolicy: + description: 'Restart policy for all containers within + the pod. One of Always, OnFailure, Never. In some + contexts, only a subset of those values may be permitted. + Default to Always. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy' + type: string + runtimeClassName: + description: 'RuntimeClassName refers to a RuntimeClass + object in the node.k8s.io group, which should be used + to run this pod. If no RuntimeClass resource matches + the named class, the pod will not be run. If unset + or empty, the "legacy" RuntimeClass will be used, + which is an implicit class with an empty definition + that uses the default runtime handler. More info: + https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class' + type: string + schedulerName: + description: If specified, the pod will be dispatched + by specified scheduler. If not specified, the pod + will be dispatched by default scheduler. + type: string + schedulingGates: + description: "SchedulingGates is an opaque list of values + that if specified will block scheduling the pod. If + schedulingGates is not empty, the pod will stay in + the SchedulingGated state and the scheduler will not + attempt to schedule the pod. \n SchedulingGates can + only be set at pod creation time, and be removed only + afterwards. \n This is a beta feature enabled by the + PodSchedulingReadiness feature gate." + items: + description: PodSchedulingGate is associated to a + Pod to guard its scheduling. + properties: + name: + description: Name of the scheduling gate. Each + scheduling gate must have a unique name field. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + securityContext: + description: 'SecurityContext holds pod-level security + attributes and common container settings. Optional: + Defaults to empty. See type description for default + values of each field.' + properties: + fsGroup: + description: "A special supplemental group that + applies to all containers in a pod. Some volume + types allow the Kubelet to change the ownership + of that volume to be owned by the pod: \n 1. The + owning GID will be the FSGroup 2. The setgid bit + is set (new files created in the volume will be + owned by FSGroup) 3. The permission bits are OR'd + with rw-rw---- \n If unset, the Kubelet will not + modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name + is windows." + format: int64 + type: integer + fsGroupChangePolicy: + description: 'fsGroupChangePolicy defines behavior + of changing ownership and permission of the volume + before being exposed inside Pod. This field will + only apply to volume types which support fsGroup + based ownership(and permissions). It will have + no effect on ephemeral volume types such as: secret, + configmaps and emptydir. Valid values are "OnRootMismatch" + and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name + is windows.' + type: string + runAsGroup: + description: The GID to run the entrypoint of the + container process. Uses runtime default if unset. + May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence + for that container. Note that this field cannot + be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run + as a non-root user. If true, the Kubelet will + validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start + the container if it does. If unset or false, no + such validation will be performed. May also be + set in SecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in + SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the + container process. Defaults to user specified + in image metadata if unspecified. May also be + set in SecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in + SecurityContext takes precedence for that container. + Note that this field cannot be set when spec.os.name + is windows. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to + all containers. If unspecified, the container + runtime will allocate a random SELinux context + for each container. May also be set in SecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence + for that container. Note that this field cannot + be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that + applies to the container. + type: string + role: + description: Role is a SELinux role label that + applies to the container. + type: string + type: + description: Type is a SELinux type label that + applies to the container. + type: string + user: + description: User is a SELinux user label that + applies to the container. + type: string + type: object + seccompProfile: + description: The seccomp options to use by the containers + in this pod. Note that this field cannot be set + when spec.os.name is windows. + properties: + localhostProfile: + description: localhostProfile indicates a profile + defined in a file on the node should be used. + The profile must be preconfigured on the node + to work. Must be a descending path, relative + to the kubelet's configured seccomp profile + location. Must only be set if type is "Localhost". + type: string + type: + description: "type indicates which kind of seccomp + profile will be applied. Valid options are: + \n Localhost - a profile defined in a file + on the node should be used. RuntimeDefault + - the container runtime default profile should + be used. Unconfined - no profile should be + applied." + type: string + required: + - type + type: object + supplementalGroups: + description: A list of groups applied to the first + process run in each container, in addition to + the container's primary GID, the fsGroup (if specified), + and group memberships defined in the container + image for the uid of the container process. If + unspecified, no additional groups are added to + any container. Note that group memberships defined + in the container image for the uid of the container + process are still effective, even if they are + not included in this list. Note that this field + cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + sysctls: + description: Sysctls hold a list of namespaced sysctls + used for the pod. Pods with unsupported sysctls + (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name + is windows. + items: + description: Sysctl defines a kernel parameter + to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + windowsOptions: + description: The Windows specific settings applied + to all containers. If unspecified, the options + within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name + is linux. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the + GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential + spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name + of the GMSA credential spec to use. + type: string + hostProcess: + description: HostProcess determines if a container + should be run as a 'Host Process' container. + This field is alpha-level and will only be + honored by components that enable the WindowsHostProcessContainers + feature flag. Setting this field without the + feature flag will result in errors when validating + the Pod. All of a Pod's containers must have + the same effective HostProcess value (it is + not allowed to have a mix of HostProcess containers + and non-HostProcess containers). In addition, + if HostProcess is true then HostNetwork must + also be set to true. + type: boolean + runAsUserName: + description: The UserName in Windows to run + the entrypoint of the container process. Defaults + to the user specified in image metadata if + unspecified. May also be set in PodSecurityContext. + If set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. + type: string + type: object + type: object + serviceAccount: + description: 'DeprecatedServiceAccount is a depreciated + alias for ServiceAccountName. Deprecated: Use serviceAccountName + instead.' + type: string + serviceAccountName: + description: 'ServiceAccountName is the name of the + ServiceAccount to use to run this pod. More info: + https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/' + type: string + setHostnameAsFQDN: + description: If true the pod's hostname will be configured + as the pod's FQDN, rather than the leaf name (the + default). In Linux containers, this means setting + the FQDN in the hostname field of the kernel (the + nodename field of struct utsname). In Windows containers, + this means setting the registry value of hostname + for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters + to FQDN. If a pod does not have FQDN, this has no + effect. Default to false. + type: boolean + shareProcessNamespace: + description: 'Share a single process namespace between + all of the containers in a pod. When this is set containers + will be able to view and signal processes from other + containers in the same pod, and the first process + in each container will not be assigned PID 1. HostPID + and ShareProcessNamespace cannot both be set. Optional: + Default to false.' + type: boolean + subdomain: + description: If specified, the fully qualified Pod hostname + will be "...svc.". If not specified, the pod will not have + a domainname at all. + type: string + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully. May be decreased in delete + request. Value must be non-negative integer. The value + zero indicates stop immediately via the kill signal + (no opportunity to shut down). If this value is nil, + the default grace period will be used instead. The + grace period is the duration in seconds after the + processes running in the pod are sent a termination + signal and the time when the processes are forcibly + halted with a kill signal. Set this value longer than + the expected cleanup time for your process. Defaults + to 30 seconds. + format: int64 + type: integer + tolerations: + description: If specified, the pod's tolerations. + items: + description: The pod this Toleration is attached to + tolerates any taint that matches the triple + using the matching operator . + properties: + effect: + description: Effect indicates the taint effect + to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, + PreferNoSchedule and NoExecute. + type: string + key: + description: Key is the taint key that the toleration + applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; + this combination means to match all values and + all keys. + type: string + operator: + description: Operator represents a key's relationship + to the value. Valid operators are Exists and + Equal. Defaults to Equal. Exists is equivalent + to wildcard for value, so that a pod can tolerate + all taints of a particular category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the + period of time the toleration (which must be + of effect NoExecute, otherwise this field is + ignored) tolerates the taint. By default, it + is not set, which means tolerate the taint forever + (do not evict). Zero and negative values will + be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: Value is the taint value the toleration + matches to. If the operator is Exists, the value + should be empty, otherwise just a regular string. + type: string + type: object + type: array + topologySpreadConstraints: + description: TopologySpreadConstraints describes how + a group of pods ought to spread across topology domains. + Scheduler will schedule pods in a way which abides + by the constraints. All topologySpreadConstraints + are ANDed. + items: + description: TopologySpreadConstraint specifies how + to spread matching pods among the given topology. + properties: + labelSelector: + description: LabelSelector is used to find matching + pods. Pods that match this label selector are + counted to determine the number of pods in their + corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: "MatchLabelKeys is a set of pod label + keys to select the pods over which spreading + will be calculated. The keys are used to lookup + values from the incoming pod labels, those key-value + labels are ANDed with labelSelector to select + the group of existing pods over which spreading + will be calculated for the incoming pod. The + same key is forbidden to exist in both MatchLabelKeys + and LabelSelector. MatchLabelKeys cannot be + set when LabelSelector isn't set. Keys that + don't exist in the incoming pod labels will + be ignored. A null or empty list means only + match against labelSelector. \n This is a beta + field and requires the MatchLabelKeysInPodTopologySpread + feature gate to be enabled (enabled by default)." + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: 'MaxSkew describes the degree to + which pods may be unevenly distributed. When + `whenUnsatisfiable=DoNotSchedule`, it is the + maximum permitted difference between the number + of matching pods in the target topology and + the global minimum. The global minimum is the + minimum number of matching pods in an eligible + domain or zero if the number of eligible domains + is less than MinDomains. For example, in a 3-zone + cluster, MaxSkew is set to 1, and pods with + the same labelSelector spread as 2/2/1: In this + case, the global minimum is 1. | zone1 | zone2 + | zone3 | | P P | P P | P | - if MaxSkew + is 1, incoming pod can only be scheduled to + zone3 to become 2/2/2; scheduling it onto zone1(zone2) + would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). - if MaxSkew is 2, incoming + pod can be scheduled onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, + it is used to give higher precedence to topologies + that satisfy it. It''s a required field. Default + value is 1 and 0 is not allowed.' + format: int32 + type: integer + minDomains: + description: "MinDomains indicates a minimum number + of eligible domains. When the number of eligible + domains with matching topology keys is less + than minDomains, Pod Topology Spread treats + \"global minimum\" as 0, and then the calculation + of Skew is performed. And when the number of + eligible domains with matching topology keys + equals or greater than minDomains, this value + has no effect on scheduling. As a result, when + the number of eligible domains is less than + minDomains, scheduler won't schedule more than + maxSkew Pods to those domains. If value is nil, + the constraint behaves as if MinDomains is equal + to 1. Valid values are integers greater than + 0. When value is not nil, WhenUnsatisfiable + must be DoNotSchedule. \n For example, in a + 3-zone cluster, MaxSkew is set to 2, MinDomains + is set to 5 and pods with the same labelSelector + spread as 2/2/2: | zone1 | zone2 | zone3 | | + \ P P | P P | P P | The number of domains + is less than 5(MinDomains), so \"global minimum\" + is treated as 0. In this situation, new pod + with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new + Pod is scheduled to any of the three zones, + it will violate MaxSkew. \n This is a beta field + and requires the MinDomainsInPodTopologySpread + feature gate to be enabled (enabled by default)." + format: int32 + type: integer + nodeAffinityPolicy: + description: "NodeAffinityPolicy indicates how + we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options + are: - Honor: only nodes matching nodeAffinity/nodeSelector + are included in the calculations. - Ignore: + nodeAffinity/nodeSelector are ignored. All nodes + are included in the calculations. \n If this + value is nil, the behavior is equivalent to + the Honor policy. This is a beta-level feature + default enabled by the NodeInclusionPolicyInPodTopologySpread + feature flag." + type: string + nodeTaintsPolicy: + description: "NodeTaintsPolicy indicates how we + will treat node taints when calculating pod + topology spread skew. Options are: - Honor: + nodes without taints, along with tainted nodes + for which the incoming pod has a toleration, + are included. - Ignore: node taints are ignored. + All nodes are included. \n If this value is + nil, the behavior is equivalent to the Ignore + policy. This is a beta-level feature default + enabled by the NodeInclusionPolicyInPodTopologySpread + feature flag." + type: string + topologyKey: + description: TopologyKey is the key of node labels. + Nodes that have a label with this key and identical + values are considered to be in the same topology. + We consider each as a "bucket", + and try to put balanced number of pods into + each bucket. We define a domain as a particular + instance of a topology. Also, we define an eligible + domain as a domain whose nodes meet the requirements + of nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", + each Node is a domain of that topology. And, + if TopologyKey is "topology.kubernetes.io/zone", + each zone is a domain of that topology. It's + a required field. + type: string + whenUnsatisfiable: + description: 'WhenUnsatisfiable indicates how + to deal with a pod if it doesn''t satisfy the + spread constraint. - DoNotSchedule (default) + tells the scheduler not to schedule it. - ScheduleAnyway + tells the scheduler to schedule the pod in any + location, but giving higher precedence to topologies + that would help reduce the skew. A constraint + is considered "Unsatisfiable" for an incoming + pod if and only if every possible node assignment + for that pod would violate "MaxSkew" on some + topology. For example, in a 3-zone cluster, + MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: | zone1 | zone2 + | zone3 | | P P P | P | P | If WhenUnsatisfiable + is set to DoNotSchedule, incoming pod can only + be scheduled to zone2(zone3) to become 3/2/1(3/1/2) + as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can + still be imbalanced, but scheduler won''t make + it *more* imbalanced. It''s a required field.' + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + description: 'List of volumes that can be mounted by + containers belonging to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes' + items: + description: Volume represents a named volume in a + pod that may be accessed by any container in the + pod. + properties: + awsElasticBlockStore: + description: 'awsElasticBlockStore represents + an AWS Disk resource that is attached to a kubelet''s + host machine and then exposed to the pod. More + info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + properties: + fsType: + description: 'fsType is the filesystem type + of the volume that you want to mount. Tip: + Ensure that the filesystem type is supported + by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. More info: + https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + TODO: how do we prevent errors in the filesystem + from compromising the machine' + type: string + partition: + description: 'partition is the partition in + the volume that you want to mount. If omitted, + the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify + the partition as "1". Similarly, the volume + partition for /dev/sda is "0" (or you can + leave the property empty).' + format: int32 + type: integer + readOnly: + description: 'readOnly value true will force + the readOnly setting in VolumeMounts. More + info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: boolean + volumeID: + description: 'volumeID is unique ID of the + persistent disk resource in AWS (Amazon + EBS volume). More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: string + required: + - volumeID + type: object + azureDisk: + description: azureDisk represents an Azure Data + Disk mount on the host and bind mount to the + pod. + properties: + cachingMode: + description: 'cachingMode is the Host Caching + mode: None, Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data + disk in the blob storage + type: string + diskURI: + description: diskURI is the URI of data disk + in the blob storage + type: string + fsType: + description: fsType is Filesystem type to + mount. Must be a filesystem type supported + by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be + "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: + multiple blob disks per storage account Dedicated: + single blob disk per storage account Managed: + azure managed data disk (only in managed + availability set). defaults to shared' + type: string + readOnly: + description: readOnly Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting + in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure File + Service mount on the host and bind mount to + the pod. + properties: + readOnly: + description: readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting + in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret + that contains Azure Storage Account Name + and Key + type: string + shareName: + description: shareName is the azure share + Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: cephFS represents a Ceph FS mount + on the host that shares a pod's lifetime + properties: + monitors: + description: 'monitors is Required: Monitors + is a collection of Ceph monitors More info: + https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + items: + type: string + type: array + path: + description: 'path is Optional: Used as the + mounted root, rather than the full Ceph + tree, default is /' + type: string + readOnly: + description: 'readOnly is Optional: Defaults + to false (read/write). ReadOnly here will + force the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: boolean + secretFile: + description: 'secretFile is Optional: SecretFile + is the path to key ring for User, default + is /etc/ceph/user.secret More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + secretRef: + description: 'secretRef is Optional: SecretRef + is reference to the authentication secret + for User, default is empty. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: 'user is optional: User is the + rados user name, default is admin More info: + https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + required: + - monitors + type: object + cinder: + description: 'cinder represents a cinder volume + attached and mounted on kubelets host machine. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + properties: + fsType: + description: 'fsType is the filesystem type + to mount. Must be a filesystem type supported + by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. More info: + https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + readOnly: + description: 'readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting + in VolumeMounts. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: boolean + secretRef: + description: 'secretRef is optional: points + to a secret object containing parameters + used to connect to OpenStack.' + properties: + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: 'volumeID used to identify the + volume in cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap + that should populate this volume + properties: + defaultMode: + description: 'defaultMode is optional: mode + bits used to set permissions on created + files by default. Must be an octal value + between 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts both octal + and decimal values, JSON requires decimal + values for mode bits. Defaults to 0644. + Directories within the path are not affected + by this setting. This might be in conflict + with other options that affect the file + mode, like fsGroup, and the result can be + other mode bits set.' + format: int32 + type: integer + items: + description: items if unspecified, each key-value + pair in the Data field of the referenced + ConfigMap will be projected into the volume + as a file whose name is the key and content + is the value. If specified, the listed keys + will be projected into the specified paths, + and unlisted keys will not be present. If + a key is specified which is not present + in the ConfigMap, the volume setup will + error unless it is marked optional. Paths + must be relative and may not contain the + '..' path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: 'mode is Optional: mode + bits used to set permissions on this + file. Must be an octal value between + 0000 and 0777 or a decimal value between + 0 and 511. YAML accepts both octal + and decimal values, JSON requires + decimal values for mode bits. If not + specified, the volume defaultMode + will be used. This might be in conflict + with other options that affect the + file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + path: + description: path is the relative path + of the file to map the key to. May + not be an absolute path. May not contain + the path element '..'. May not start + with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: optional specify whether the + ConfigMap or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) + represents ephemeral storage that is handled + by certain external CSI drivers (Beta feature). + properties: + driver: + description: driver is the name of the CSI + driver that handles this volume. Consult + with your admin for the correct name as + registered in the cluster. + type: string + fsType: + description: fsType to mount. Ex. "ext4", + "xfs", "ntfs". If not provided, the empty + value is passed to the associated CSI driver + which will determine the default filesystem + to apply. + type: string + nodePublishSecretRef: + description: nodePublishSecretRef is a reference + to the secret object containing sensitive + information to pass to the CSI driver to + complete the CSI NodePublishVolume and NodeUnpublishVolume + calls. This field is optional, and may + be empty if no secret is required. If the + secret object contains more than one secret, + all secret references are passed. + properties: + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: readOnly specifies a read-only + configuration for the volume. Defaults to + false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: volumeAttributes stores driver-specific + properties that are passed to the CSI driver. + Consult your driver's documentation for + supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API + about the pod that should populate this volume + properties: + defaultMode: + description: 'Optional: mode bits to use on + created files by default. Must be a Optional: + mode bits used to set permissions on created + files by default. Must be an octal value + between 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts both octal + and decimal values, JSON requires decimal + values for mode bits. Defaults to 0644. + Directories within the path are not affected + by this setting. This might be in conflict + with other options that affect the file + mode, like fsGroup, and the result can be + other mode bits set.' + format: int32 + type: integer + items: + description: Items is a list of downward API + volume file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a field + of the pod: only annotations, labels, + name and namespace are supported.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: 'Optional: mode bits used + to set permissions on this file, must + be an octal value between 0000 and + 0777 or a decimal value between 0 + and 511. YAML accepts both octal and + decimal values, JSON requires decimal + values for mode bits. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, + like fsGroup, and the result can be + other mode bits set.' + format: int32 + type: integer + path: + description: 'Required: Path is the + relative path name of the file to + be created. Must not be absolute or + contain the ''..'' path. Must be utf-8 + encoded. The first item of the relative + path must not start with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of + the container: only resources limits + and requests (limits.cpu, limits.memory, + requests.cpu and requests.memory) + are currently supported.' + properties: + containerName: + description: 'Container name: required + for volumes, optional for env + vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + emptyDir: + description: 'emptyDir represents a temporary + directory that shares a pod''s lifetime. More + info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + properties: + medium: + description: 'medium represents what type + of storage medium should back this directory. + The default is "" which means to use the + node''s default medium. Must be an empty + string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: 'sizeLimit is the total amount + of local storage required for this EmptyDir + volume. The size limit is also applicable + for memory medium. The maximum usage on + memory medium EmptyDir would be the minimum + value between the SizeLimit specified here + and the sum of memory limits of all containers + in a pod. The default is nil which means + that the limit is undefined. More info: + https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: "ephemeral represents a volume that + is handled by a cluster storage driver. The + volume's lifecycle is tied to the pod that defines + it - it will be created before the pod starts, + and deleted when the pod is removed. \n Use + this if: a) the volume is only needed while + the pod runs, b) features of normal volumes + like restoring from snapshot or capacity tracking + are needed, c) the storage driver is specified + through a storage class, and d) the storage + driver supports dynamic volume provisioning + through a PersistentVolumeClaim (see EphemeralVolumeSource + for more information on the connection between + this volume type and PersistentVolumeClaim). + \n Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than + the lifecycle of an individual pod. \n Use CSI + for light-weight local ephemeral volumes if + the CSI driver is meant to be used that way + - see the documentation of the driver for more + information. \n A pod can use both types of + ephemeral volumes and persistent volumes at + the same time." + properties: + volumeClaimTemplate: + description: "Will be used to create a stand-alone + PVC to provision the volume. The pod in + which this EphemeralVolumeSource is embedded + will be the owner of the PVC, i.e. the PVC + will be deleted together with the pod. The + name of the PVC will be `-` where `` is the name + from the `PodSpec.Volumes` array entry. + Pod validation will reject the pod if the + concatenated name is not valid for a PVC + (for example, too long). \n An existing + PVC with that name that is not owned by + the pod will *not* be used for the pod to + avoid using an unrelated volume by mistake. + Starting the pod is then blocked until the + unrelated PVC is removed. If such a pre-created + PVC is meant to be used by the pod, the + PVC has to updated with an owner reference + to the pod once the pod exists. Normally + this should not be necessary, but it may + be useful when manually reconstructing a + broken cluster. \n This field is read-only + and no changes will be made by Kubernetes + to the PVC after it has been created. \n + Required, must not be nil." + properties: + metadata: + description: May contain labels and annotations + that will be copied into the PVC when + creating it. No other fields are allowed + and will be rejected during validation. + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: The specification for the + PersistentVolumeClaim. The entire content + is copied unchanged into the PVC that + gets created from this template. The + same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: 'accessModes contains + the desired access modes the volume + should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'dataSource field can + be used to specify either: * An + existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external + controller can support the specified + data source, it will create a new + volume based on the contents of + the specified data source. When + the AnyVolumeDataSource feature + gate is enabled, dataSource contents + will be copied to dataSourceRef, + and dataSourceRef contents will + be copied to dataSource when dataSourceRef.namespace + is not specified. If the namespace + is specified, then dataSourceRef + will not be copied to dataSource.' + properties: + apiGroup: + description: APIGroup is the group + for the resource being referenced. + If APIGroup is not specified, + the specified Kind must be in + the core API group. For any + other third-party types, APIGroup + is required. + type: string + kind: + description: Kind is the type + of resource being referenced + type: string + name: + description: Name is the name + of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: 'dataSourceRef specifies + the object from which to populate + the volume with data, if a non-empty + volume is desired. This may be any + object from a non-empty API group + (non core object) or a PersistentVolumeClaim + object. When this field is specified, + volume binding will only succeed + if the type of the specified object + matches some installed volume populator + or dynamic provisioner. This field + will replace the functionality of + the dataSource field and as such + if both fields are non-empty, they + must have the same value. For backwards + compatibility, when namespace isn''t + specified in dataSourceRef, both + fields (dataSource and dataSourceRef) + will be set to the same value automatically + if one of them is empty and the + other is non-empty. When namespace + is specified in dataSourceRef, dataSource + isn''t set to the same value and + must be empty. There are three important + differences between dataSource and + dataSourceRef: * While dataSource + only allows two specific types of + objects, dataSourceRef allows any + non-core object, as well as PersistentVolumeClaim + objects. * While dataSource ignores + disallowed values (dropping them), + dataSourceRef preserves all values, + and generates an error if a disallowed + value is specified. * While dataSource + only allows local objects, dataSourceRef + allows objects in any namespaces. + (Beta) Using this field requires + the AnyVolumeDataSource feature + gate to be enabled. (Alpha) Using + the namespace field of dataSourceRef + requires the CrossNamespaceVolumeDataSource + feature gate to be enabled.' + properties: + apiGroup: + description: APIGroup is the group + for the resource being referenced. + If APIGroup is not specified, + the specified Kind must be in + the core API group. For any + other third-party types, APIGroup + is required. + type: string + kind: + description: Kind is the type + of resource being referenced + type: string + name: + description: Name is the name + of resource being referenced + type: string + namespace: + description: Namespace is the + namespace of resource being + referenced Note that when a + namespace is specified, a gateway.networking.k8s.io/ReferenceGrant + object is required in the referent + namespace to allow that namespace's + owner to accept the reference. + See the ReferenceGrant documentation + for details. (Alpha) This field + requires the CrossNamespaceVolumeDataSource + feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: 'resources represents + the minimum resources the volume + should have. If RecoverVolumeExpansionFailure + feature is enabled users are allowed + to specify resource requirements + that are lower than previous value + but must still be higher than capacity + recorded in the status field of + the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + claims: + description: "Claims lists the + names of resources, defined + in spec.resourceClaims, that + are used by this container. + \n This is an alpha field and + requires enabling the DynamicResourceAllocation + feature gate. \n This field + is immutable. It can only be + set for containers." + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match + the name of one entry + in pod.spec.resourceClaims + of the Pod where this + field is used. It makes + that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes + the maximum amount of compute + resources allowed. More info: + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes + the minimum amount of compute + resources required. If Requests + is omitted for a container, + it defaults to Limits if that + is explicitly specified, otherwise + to an implementation-defined + value. Requests cannot exceed + Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: selector is a label query + over volumes to consider for binding. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: A label selector + requirement is a selector + that contains values, a key, + and an operator that relates + the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: operator represents + a key's relationship to + a set of values. Valid + operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an + array of string values. + If the operator is In + or NotIn, the values array + must be non-empty. If + the operator is Exists + or DoesNotExist, the values + array must be empty. This + array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a + map of {key,value} pairs. A + single {key,value} in the matchLabels + map is equivalent to an element + of matchExpressions, whose key + field is "key", the operator + is "In", and the values array + contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: 'storageClassName is + the name of the StorageClass required + by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what + type of volume is required by the + claim. Value of Filesystem is implied + when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding + reference to the PersistentVolume + backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource + that is attached to a kubelet's host machine + and then exposed to the pod. + properties: + fsType: + description: 'fsType is the filesystem type + to mount. Must be a filesystem type supported + by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be + "ext4" if unspecified. TODO: how do we prevent + errors in the filesystem from compromising + the machine' + type: string + lun: + description: 'lun is Optional: FC target lun + number' + format: int32 + type: integer + readOnly: + description: 'readOnly is Optional: Defaults + to false (read/write). ReadOnly here will + force the ReadOnly setting in VolumeMounts.' + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target + worldwide names (WWNs)' + items: + type: string + type: array + wwids: + description: 'wwids Optional: FC volume world + wide identifiers (wwids) Either wwids or + combination of targetWWNs and lun must be + set, but not both simultaneously.' + items: + type: string + type: array + type: object + flexVolume: + description: flexVolume represents a generic volume + resource that is provisioned/attached using + an exec based plugin. + properties: + driver: + description: driver is the name of the driver + to use for this volume. + type: string + fsType: + description: fsType is the filesystem type + to mount. Must be a filesystem type supported + by the host operating system. Ex. "ext4", + "xfs", "ntfs". The default filesystem depends + on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field + holds extra command options if any.' + type: object + readOnly: + description: 'readOnly is Optional: defaults + to false (read/write). ReadOnly here will + force the ReadOnly setting in VolumeMounts.' + type: boolean + secretRef: + description: 'secretRef is Optional: secretRef + is reference to the secret object containing + sensitive information to pass to the plugin + scripts. This may be empty if no secret + object is specified. If the secret object + contains more than one secret, all secrets + are passed to the plugin scripts.' + properties: + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: flocker represents a Flocker volume + attached to a kubelet's host machine. This depends + on the Flocker control service being running + properties: + datasetName: + description: datasetName is Name of the dataset + stored as metadata -> name on the dataset + for Flocker should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the + dataset. This is unique identifier of a + Flocker dataset + type: string + type: object + gcePersistentDisk: + description: 'gcePersistentDisk represents a GCE + Disk resource that is attached to a kubelet''s + host machine and then exposed to the pod. More + info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + properties: + fsType: + description: 'fsType is filesystem type of + the volume that you want to mount. Tip: + Ensure that the filesystem type is supported + by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. More info: + https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + TODO: how do we prevent errors in the filesystem + from compromising the machine' + type: string + partition: + description: 'partition is the partition in + the volume that you want to mount. If omitted, + the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify + the partition as "1". Similarly, the volume + partition for /dev/sda is "0" (or you can + leave the property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + format: int32 + type: integer + pdName: + description: 'pdName is unique name of the + PD resource in GCE. Used to identify the + disk in GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: string + readOnly: + description: 'readOnly here will force the + ReadOnly setting in VolumeMounts. Defaults + to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: boolean + required: + - pdName + type: object + gitRepo: + description: 'gitRepo represents a git repository + at a particular revision. DEPRECATED: GitRepo + is deprecated. To provision a container with + a git repo, mount an EmptyDir into an InitContainer + that clones the repo using git, then mount the + EmptyDir into the Pod''s container.' + properties: + directory: + description: directory is the target directory + name. Must not contain or start with '..'. If + '.' is supplied, the volume directory will + be the git repository. Otherwise, if specified, + the volume will contain the git repository + in the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for + the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: 'glusterfs represents a Glusterfs + mount on the host that shares a pod''s lifetime. + More info: https://examples.k8s.io/volumes/glusterfs/README.md' + properties: + endpoints: + description: 'endpoints is the endpoint name + that details Glusterfs topology. More info: + https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + path: + description: 'path is the Glusterfs volume + path. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + readOnly: + description: 'readOnly here will force the + Glusterfs volume to be mounted with read-only + permissions. Defaults to false. More info: + https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: 'hostPath represents a pre-existing + file or directory on the host machine that is + directly exposed to the container. This is generally + used for system agents or other privileged things + that are allowed to see the host machine. Most + containers will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + --- TODO(jonesdl) We need to restrict who can + use host directory mounts and who can/can not + mount host directories as read/write.' + properties: + path: + description: 'path of the directory on the + host. If the path is a symlink, it will + follow the link to the real path. More info: + https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + type: + description: 'type for HostPath Volume Defaults + to "" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + required: + - path + type: object + iscsi: + description: 'iscsi represents an ISCSI Disk resource + that is attached to a kubelet''s host machine + and then exposed to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether + support iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether + support iSCSI Session CHAP authentication + type: boolean + fsType: + description: 'fsType is the filesystem type + of the volume that you want to mount. Tip: + Ensure that the filesystem type is supported + by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. More info: + https://kubernetes.io/docs/concepts/storage/volumes#iscsi + TODO: how do we prevent errors in the filesystem + from compromising the machine' + type: string + initiatorName: + description: initiatorName is the custom iSCSI + Initiator Name. If initiatorName is specified + with iscsiInterface simultaneously, new + iSCSI interface : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified + Name. + type: string + iscsiInterface: + description: iscsiInterface is the interface + Name that uses an iSCSI transport. Defaults + to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun + number. + format: int32 + type: integer + portals: + description: portals is the iSCSI Target Portal + List. The portal is either an IP or ip_addr:port + if the port is other than default (typically + TCP ports 860 and 3260). + items: + type: string + type: array + readOnly: + description: readOnly here will force the + ReadOnly setting in VolumeMounts. Defaults + to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret + for iSCSI target and initiator authentication + properties: + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: targetPortal is iSCSI Target + Portal. The Portal is either an IP or ip_addr:port + if the port is other than default (typically + TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: 'name of the volume. Must be a DNS_LABEL + and unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + nfs: + description: 'nfs represents an NFS mount on the + host that shares a pod''s lifetime More info: + https://kubernetes.io/docs/concepts/storage/volumes#nfs' + properties: + path: + description: 'path that is exported by the + NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + readOnly: + description: 'readOnly here will force the + NFS export to be mounted with read-only + permissions. Defaults to false. More info: + https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: boolean + server: + description: 'server is the hostname or IP + address of the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: 'persistentVolumeClaimVolumeSource + represents a reference to a PersistentVolumeClaim + in the same namespace. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + claimName: + description: 'claimName is the name of a PersistentVolumeClaim + in the same namespace as the pod using this + volume. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + type: string + readOnly: + description: readOnly Will force the ReadOnly + setting in VolumeMounts. Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: photonPersistentDisk represents a + PhotonController persistent disk attached and + mounted on kubelets host machine + properties: + fsType: + description: fsType is the filesystem type + to mount. Must be a filesystem type supported + by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be + "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies + Photon Controller persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents a portworx + volume attached and mounted on kubelets host + machine + properties: + fsType: + description: fSType represents the filesystem + type to mount Must be a filesystem type + supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to + be "ext4" if unspecified. + type: string + readOnly: + description: readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting + in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies + a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources + secrets, configmaps, and downward API + properties: + defaultMode: + description: defaultMode are the mode bits + used to set permissions on created files + by default. Must be an octal value between + 0000 and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and decimal + values, JSON requires decimal values for + mode bits. Directories within the path are + not affected by this setting. This might + be in conflict with other options that affect + the file mode, like fsGroup, and the result + can be other mode bits set. + format: int32 + type: integer + sources: + description: sources is the list of volume + projections + items: + description: Projection that may be projected + along with other supported volume types + properties: + configMap: + description: configMap information about + the configMap data to project + properties: + items: + description: items if unspecified, + each key-value pair in the Data + field of the referenced ConfigMap + will be projected into the volume + as a file whose name is the key + and content is the value. If specified, + the listed keys will be projected + into the specified paths, and + unlisted keys will not be present. + If a key is specified which is + not present in the ConfigMap, + the volume setup will error unless + it is marked optional. Paths must + be relative and may not contain + the '..' path or start with '..'. + items: + description: Maps a string key + to a path within a volume. + properties: + key: + description: key is the key + to project. + type: string + mode: + description: 'mode is Optional: + mode bits used to set permissions + on this file. Must be an + octal value between 0000 + and 0777 or a decimal value + between 0 and 511. YAML + accepts both octal and decimal + values, JSON requires decimal + values for mode bits. If + not specified, the volume + defaultMode will be used. + This might be in conflict + with other options that + affect the file mode, like + fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + path: + description: path is the relative + path of the file to map + the key to. May not be an + absolute path. May not contain + the path element '..'. May + not start with the string + '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. + apiVersion, kind, uid?' + type: string + optional: + description: optional specify whether + the ConfigMap or its keys must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information + about the downwardAPI data to project + properties: + items: + description: Items is a list of + DownwardAPIVolume file + items: + description: DownwardAPIVolumeFile + represents information to create + the file containing the pod + field + properties: + fieldRef: + description: 'Required: Selects + a field of the pod: only + annotations, labels, name + and namespace are supported.' + properties: + apiVersion: + description: Version of + the schema the FieldPath + is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the + field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: 'Optional: mode + bits used to set permissions + on this file, must be an + octal value between 0000 + and 0777 or a decimal value + between 0 and 511. YAML + accepts both octal and decimal + values, JSON requires decimal + values for mode bits. If + not specified, the volume + defaultMode will be used. + This might be in conflict + with other options that + affect the file mode, like + fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + path: + description: 'Required: Path + is the relative path name + of the file to be created. + Must not be absolute or + contain the ''..'' path. + Must be utf-8 encoded. The + first item of the relative + path must not start with + ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource + of the container: only resources + limits and requests (limits.cpu, + limits.memory, requests.cpu + and requests.memory) are + currently supported.' + properties: + containerName: + description: 'Container + name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies + the output format of + the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: + resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + secret: + description: secret information about + the secret data to project + properties: + items: + description: items if unspecified, + each key-value pair in the Data + field of the referenced Secret + will be projected into the volume + as a file whose name is the key + and content is the value. If specified, + the listed keys will be projected + into the specified paths, and + unlisted keys will not be present. + If a key is specified which is + not present in the Secret, the + volume setup will error unless + it is marked optional. Paths must + be relative and may not contain + the '..' path or start with '..'. + items: + description: Maps a string key + to a path within a volume. + properties: + key: + description: key is the key + to project. + type: string + mode: + description: 'mode is Optional: + mode bits used to set permissions + on this file. Must be an + octal value between 0000 + and 0777 or a decimal value + between 0 and 511. YAML + accepts both octal and decimal + values, JSON requires decimal + values for mode bits. If + not specified, the volume + defaultMode will be used. + This might be in conflict + with other options that + affect the file mode, like + fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + path: + description: path is the relative + path of the file to map + the key to. May not be an + absolute path. May not contain + the path element '..'. May + not start with the string + '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. + apiVersion, kind, uid?' + type: string + optional: + description: optional field specify + whether the Secret or its key + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken is + information about the serviceAccountToken + data to project + properties: + audience: + description: audience is the intended + audience of the token. A recipient + of a token must identify itself + with an identifier specified in + the audience of the token, and + otherwise should reject the token. + The audience defaults to the identifier + of the apiserver. + type: string + expirationSeconds: + description: expirationSeconds is + the requested duration of validity + of the service account token. + As the token approaches expiration, + the kubelet volume plugin will + proactively rotate the service + account token. The kubelet will + start trying to rotate the token + if the token is older than 80 + percent of its time to live or + if the token is older than 24 + hours.Defaults to 1 hour and must + be at least 10 minutes. + format: int64 + type: integer + path: + description: path is the path relative + to the mount point of the file + to project the token into. + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + description: quobyte represents a Quobyte mount + on the host that shares a pod's lifetime + properties: + group: + description: group to map volume access to + Default is no group + type: string + readOnly: + description: readOnly here will force the + Quobyte volume to be mounted with read-only + permissions. Defaults to false. + type: boolean + registry: + description: registry represents a single + or multiple Quobyte Registry services specified + as a string as host:port pair (multiple + entries are separated with commas) which + acts as the central registry for volumes + type: string + tenant: + description: tenant owning the given Quobyte + volume in the Backend Used with dynamically + provisioned Quobyte volumes, value is set + by the plugin + type: string + user: + description: user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that references + an already created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: 'rbd represents a Rados Block Device + mount on the host that shares a pod''s lifetime. + More info: https://examples.k8s.io/volumes/rbd/README.md' + properties: + fsType: + description: 'fsType is the filesystem type + of the volume that you want to mount. Tip: + Ensure that the filesystem type is supported + by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. More info: + https://kubernetes.io/docs/concepts/storage/volumes#rbd + TODO: how do we prevent errors in the filesystem + from compromising the machine' + type: string + image: + description: 'image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + keyring: + description: 'keyring is the path to key ring + for RBDUser. Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + monitors: + description: 'monitors is a collection of + Ceph monitors. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + items: + type: string + type: array + pool: + description: 'pool is the rados pool name. + Default is rbd. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + readOnly: + description: 'readOnly here will force the + ReadOnly setting in VolumeMounts. Defaults + to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: boolean + secretRef: + description: 'secretRef is name of the authentication + secret for RBDUser. If provided overrides + keyring. Default is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: 'user is the rados user name. + Default is admin. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO persistent + volume attached and mounted on Kubernetes nodes. + properties: + fsType: + description: fsType is the filesystem type + to mount. Must be a filesystem type supported + by the host operating system. Ex. "ext4", + "xfs", "ntfs". Default is "xfs". + type: string + gateway: + description: gateway is the host address of + the ScaleIO API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name + of the ScaleIO Protection Domain for the + configured storage. + type: string + readOnly: + description: readOnly Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting + in VolumeMounts. + type: boolean + secretRef: + description: secretRef references to the secret + for ScaleIO user and other sensitive information. + If this is not provided, Login operation + will fail. + properties: + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable + SSL communication with Gateway, default + false + type: boolean + storageMode: + description: storageMode indicates whether + the storage for a volume should be ThickProvisioned + or ThinProvisioned. Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage + Pool associated with the protection domain. + type: string + system: + description: system is the name of the storage + system as configured in ScaleIO. + type: string + volumeName: + description: volumeName is the name of a volume + already created in the ScaleIO system that + is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: 'secret represents a secret that + should populate this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + properties: + defaultMode: + description: 'defaultMode is Optional: mode + bits used to set permissions on created + files by default. Must be an octal value + between 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts both octal + and decimal values, JSON requires decimal + values for mode bits. Defaults to 0644. + Directories within the path are not affected + by this setting. This might be in conflict + with other options that affect the file + mode, like fsGroup, and the result can be + other mode bits set.' + format: int32 + type: integer + items: + description: items If unspecified, each key-value + pair in the Data field of the referenced + Secret will be projected into the volume + as a file whose name is the key and content + is the value. If specified, the listed keys + will be projected into the specified paths, + and unlisted keys will not be present. If + a key is specified which is not present + in the Secret, the volume setup will error + unless it is marked optional. Paths must + be relative and may not contain the '..' + path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: 'mode is Optional: mode + bits used to set permissions on this + file. Must be an octal value between + 0000 and 0777 or a decimal value between + 0 and 511. YAML accepts both octal + and decimal values, JSON requires + decimal values for mode bits. If not + specified, the volume defaultMode + will be used. This might be in conflict + with other options that affect the + file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + path: + description: path is the relative path + of the file to map the key to. May + not be an absolute path. May not contain + the path element '..'. May not start + with the string '..'. + type: string + required: + - key + - path + type: object + type: array + optional: + description: optional field specify whether + the Secret or its keys must be defined + type: boolean + secretName: + description: 'secretName is the name of the + secret in the pod''s namespace to use. More + info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + type: string + type: object + storageos: + description: storageOS represents a StorageOS + volume attached and mounted on Kubernetes nodes. + properties: + fsType: + description: fsType is the filesystem type + to mount. Must be a filesystem type supported + by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be + "ext4" if unspecified. + type: string + readOnly: + description: readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting + in VolumeMounts. + type: boolean + secretRef: + description: secretRef specifies the secret + to use for obtaining the StorageOS API credentials. If + not specified, default values will be attempted. + properties: + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: volumeName is the human-readable + name of the StorageOS volume. Volume names + are only unique within a namespace. + type: string + volumeNamespace: + description: volumeNamespace specifies the + scope of the volume within StorageOS. If + no namespace is specified then the Pod's + namespace will be used. This allows the + Kubernetes name scoping to be mirrored within + StorageOS for tighter integration. Set VolumeName + to any name to override the default behaviour. + Set to "default" if you are not using namespaces + within StorageOS. Namespaces that do not + pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: vsphereVolume represents a vSphere + volume attached and mounted on kubelets host + machine + properties: + fsType: + description: fsType is filesystem type to + mount. Must be a filesystem type supported + by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be + "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage + Policy Based Management (SPBM) profile ID + associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage + Policy Based Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies + vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + required: + - containers + type: object + type: object + type: object + description: MPIReplicaSpecs contains maps from `MPIReplicaType` to + `ReplicaSpec` that specify the MPI replicas to run. + type: object + runPolicy: + description: RunPolicy encapsulates various runtime policies of the + job. + properties: + activeDeadlineSeconds: + description: Specifies the duration in seconds relative to the + startTime that the job may be active before the system tries + to terminate it; value must be positive integer. + format: int64 + type: integer + backoffLimit: + description: Optional number of retries before marking this job + failed. + format: int32 + type: integer + cleanPodPolicy: + description: CleanPodPolicy defines the policy to kill pods after + the job completes. Default to Running. + type: string + schedulingPolicy: + description: SchedulingPolicy defines the policy related to scheduling, + e.g. gang-scheduling + properties: + minAvailable: + description: "MinAvailable defines the minimal number of member + to run the PodGroup. If the gang-scheduling isn't empty, + input is passed to `.spec.minMember` in PodGroup. Note that, + when using this field, you need to make sure the application + supports resizing (e.g., Elastic Horovod). \n If not set, + it defaults to the number of workers." + format: int32 + type: integer + minResources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: MinResources defines the minimal resources of + members to run the PodGroup. If the gang-scheduling isn't + empty, input is passed to `.spec.minResources` in PodGroup + for scheduler-plugins. + type: object + priorityClass: + description: PriorityClass defines the PodGroup's PriorityClass. + If the gang-scheduling is set to the volcano, input is passed + to `.spec.priorityClassName` in PodGroup for volcano, and + if it is set to the scheduler-plugins, input isn't passed + to PodGroup for scheduler-plugins. + type: string + queue: + description: Queue defines the queue name to allocate resource + for PodGroup. If the gang-scheduling is set to the volcano, + input is passed to `.spec.queue` in PodGroup for the volcano, + and if it is set to the scheduler-plugins, input isn't passed + to PodGroup. + type: string + scheduleTimeoutSeconds: + description: SchedulerTimeoutSeconds defines the maximal time + of members to wait before run the PodGroup. If the gang-scheduling + is set to the scheduler-plugins, input is passed to `.spec.scheduleTimeoutSeconds` + in PodGroup for the scheduler-plugins, and if it is set + to the volcano, input isn't passed to PodGroup. + format: int32 + type: integer + type: object + suspend: + default: false + description: "suspend specifies whether the MPIJob controller + should create Pods or not. If a MPIJob is created with suspend + set to true, no Pods are created by the MPIJob controller. If + a MPIJob is suspended after creation (i.e. the flag goes from + false to true), the MPIJob controller will delete all active + Pods and PodGroups associated with this MPIJob. Also, it will + suspend the Launcher Job. Users must design their workload to + gracefully handle this. Suspending a Job will reset the StartTime + field of the MPIJob. \n Defaults to false." + type: boolean + ttlSecondsAfterFinished: + description: TTLSecondsAfterFinished is the TTL to clean up jobs. + It may take extra ReconcilePeriod seconds for the cleanup, since + reconcile gets called periodically. Default to infinite. + format: int32 + type: integer + type: object + slotsPerWorker: + default: 1 + description: Specifies the number of slots per worker used in hostfile. + Defaults to 1. + format: int32 + type: integer + sshAuthMountPath: + default: /root/.ssh + description: SSHAuthMountPath is the directory where SSH keys are + mounted. Defaults to "/root/.ssh". + type: string + required: + - mpiReplicaSpecs + type: object + status: + description: JobStatus represents the current observed state of the training + Job. + properties: + completionTime: + description: Represents time when the job was completed. It is not + guaranteed to be set in happens-before order across separate operations. + It is represented in RFC3339 form and is in UTC. + format: date-time + type: string + conditions: + description: conditions is a list of current observed job conditions. + items: + description: JobCondition describes the state of the job at a certain + point. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + format: date-time + type: string + lastUpdateTime: + description: The last time this condition was updated. + format: date-time + type: string + message: + description: A human-readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of job condition. + type: string + required: + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + lastReconcileTime: + description: Represents last time when the job was reconciled. It + is not guaranteed to be set in happens-before order across separate + operations. It is represented in RFC3339 form and is in UTC. + format: date-time + type: string + replicaStatuses: + additionalProperties: + description: ReplicaStatus represents the current observed state + of the replica. + properties: + active: + description: The number of actively running pods. + format: int32 + type: integer + failed: + description: The number of pods which reached phase failed. + format: int32 + type: integer + labelSelector: + description: 'Deprecated: Use selector instead' + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists or + DoesNotExist, the values array must be empty. This + array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is + "key", the operator is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + selector: + description: A selector is a label query over a set of resources. + The result of matchLabels and matchExpressions are ANDed. + An empty selector matches all objects. A null selector matches + no objects. + type: string + succeeded: + description: The number of pods which reached phase succeeded. + format: int32 + type: integer + type: object + description: replicaStatuses is map of ReplicaType and ReplicaStatus, + specifies the status of each replica. + type: object + startTime: + description: Represents time when the job was acknowledged by the + job controller. It is not guaranteed to be set in happens-before + order across separate operations. It is represented in RFC3339 form + and is in UTC. + format: date-time + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: mpi-operator + app.kubernetes.io/component: mpijob + app.kubernetes.io/name: mpi-operator + kustomize.component: mpi-operator + name: mpi-operator + namespace: mpi-operator +--- +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-mpijobs-admin: "true" +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: mpi-operator + app.kubernetes.io/component: mpijob + app.kubernetes.io/name: mpi-operator + kustomize.component: mpi-operator + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-admin: "true" + name: kubeflow-mpijobs-admin +rules: [] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: mpi-operator + app.kubernetes.io/component: mpijob + app.kubernetes.io/name: mpi-operator + kustomize.component: mpi-operator + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-edit: "true" + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-mpijobs-admin: "true" + name: kubeflow-mpijobs-edit +rules: +- apiGroups: + - kubeflow.org + resources: + - mpijobs + - mpijobs/status + verbs: + - get + - list + - watch + - create + - delete + - deletecollection + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: mpi-operator + app.kubernetes.io/component: mpijob + app.kubernetes.io/name: mpi-operator + kustomize.component: mpi-operator + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-view: "true" + name: kubeflow-mpijobs-view +rules: +- apiGroups: + - kubeflow.org + resources: + - mpijobs + - mpijobs/status + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: mpi-operator + app.kubernetes.io/component: mpijob + app.kubernetes.io/name: mpi-operator + kustomize.component: mpi-operator + name: mpi-operator +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + - services + verbs: + - create + - list + - watch + - update +- apiGroups: + - "" + resources: + - pods + verbs: + - create + - get + - list + - watch + - delete + - update + - patch +- apiGroups: + - "" + resources: + - pods/exec + verbs: + - create +- apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - update +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - list + - update + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get +- apiGroups: + - kubeflow.org + resources: + - mpijobs + - mpijobs/finalizers + - mpijobs/status + verbs: + - '*' +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - '*' +- apiGroups: + - scheduling.incubator.k8s.io + - scheduling.sigs.dev + - scheduling.volcano.sh + resources: + - queues + - podgroups + verbs: + - '*' +- apiGroups: + - scheduling.x-k8s.io + resources: + - podgroups + verbs: + - '*' +- apiGroups: + - scheduling.k8s.io + resources: + - priorityclasses + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: mpi-operator + app.kubernetes.io/component: mpijob + app.kubernetes.io/name: mpi-operator + kustomize.component: mpi-operator + name: mpi-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: mpi-operator +subjects: +- kind: ServiceAccount + name: mpi-operator + namespace: mpi-operator +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: mpi-operator + app.kubernetes.io/component: mpijob + app.kubernetes.io/name: mpi-operator + kustomize.component: mpi-operator + name: mpi-operator + namespace: mpi-operator +spec: + replicas: 1 + selector: + matchLabels: + app: mpi-operator + app.kubernetes.io/component: mpijob + app.kubernetes.io/name: mpi-operator + kustomize.component: mpi-operator + template: + metadata: + annotations: + sidecar.istio.io/inject: "false" + labels: + app: mpi-operator + app.kubernetes.io/component: mpijob + app.kubernetes.io/name: mpi-operator + kustomize.component: mpi-operator + spec: + containers: + - args: + - -alsologtostderr + image: mpioperator/mpi-operator:master + name: mpi-operator + serviceAccountName: mpi-operator diff --git a/e2e2/test/cases/nvidia/manifests/nvidia-device-plugin.yaml b/e2e2/test/cases/nvidia/manifests/nvidia-device-plugin.yaml new file mode 100644 index 000000000..88e2c07a6 --- /dev/null +++ b/e2e2/test/cases/nvidia/manifests/nvidia-device-plugin.yaml @@ -0,0 +1,56 @@ +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: nvidia-device-plugin-daemonset + namespace: kube-system +spec: + selector: + matchLabels: + name: nvidia-device-plugin-ds + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + name: nvidia-device-plugin-ds + spec: + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + # Mark this pod as a critical add-on; when enabled, the critical add-on + # scheduler reserves resources for critical add-on pods so that they can + # be rescheduled after a failure. + # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ + priorityClassName: "system-node-critical" + containers: + - image: nvcr.io/nvidia/k8s-device-plugin:v0.14.2 + name: nvidia-device-plugin-ctr + env: + - name: FAIL_ON_INIT_ERROR + value: "false" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + volumeMounts: + - name: device-plugin + mountPath: /var/lib/kubelet/device-plugins + volumes: + - name: device-plugin + hostPath: + path: /var/lib/kubelet/device-plugins diff --git a/e2e2/test/cases/nvidia/mpi_test.go b/e2e2/test/cases/nvidia/mpi_test.go new file mode 100644 index 000000000..90a32b206 --- /dev/null +++ b/e2e2/test/cases/nvidia/mpi_test.go @@ -0,0 +1,124 @@ +package nvidia + +import ( + "context" + "slices" + "testing" + "time" + + fwext "github.com/aws/aws-k8s-tester/e2e2/internal/framework_extensions" + kubeflowv2beta1 "github.com/kubeflow/mpi-operator/pkg/apis/kubeflow/v2beta1" + "sigs.k8s.io/e2e-framework/klient/k8s" + "sigs.k8s.io/e2e-framework/klient/wait" + "sigs.k8s.io/e2e-framework/klient/wait/conditions" + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/features" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestMPIJobPytorchTraining(t *testing.T) { + manifestSingleNode := "manifests/mpi-job-pytorch-training-single-node.yaml" + + singleNode := features.New("single-node"). + WithLabel("suite", "nvidia"). + WithLabel("hardware", "gpu"). + Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + err := fwext.ApplyFile(cfg.Client().RESTConfig(), manifestSingleNode) + if err != nil { + t.Fatal(err) + } + return ctx + }). + Assess("MPIJob succeeds", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + rsrc := cfg.Client().Resources() + if err := kubeflowv2beta1.AddToScheme(rsrc.GetScheme()); err != nil { + t.Fatal(err) + } + j := kubeflowv2beta1.MPIJob{ + ObjectMeta: metav1.ObjectMeta{Name: "pytorch-training-single-node", Namespace: "default"}, + } + timeout := time.Minute * 20 + err := wait.For(fwext.NewConditionExtension(rsrc).ResourceMatch(&j, mpiJobSucceeded), + wait.WithTimeout(timeout)) + if err != nil { + t.Fatal(err) + } + return ctx + }). + Teardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + err := fwext.DeleteFile(cfg.Client().RESTConfig(), manifestSingleNode) + if err != nil { + t.Fatal(err) + } + return ctx + }). + Feature() + + manifestsMultiNode := []string{ + "manifests/efa-device-plugin.yaml", + "manifests/mpi-job-pytorch-training-multi-node.yaml", + } + + multiNode := features.New("multi-node"). + WithLabel("suite", "nvidia"). + WithLabel("hardware", "gpu"). + WithLabel("hardware", "efa"). + Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + err := fwext.ApplyFiles(cfg.Client().RESTConfig(), manifestsMultiNode) + if err != nil { + t.Fatal(err) + } + return ctx + }). + Assess("EFA device plugin daemonset ready", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + ds := appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Name: "efa-device-plugin-daemonset", Namespace: "kube-system"}, + } + err := wait.For(fwext.NewConditionExtension(cfg.Client().Resources()).DaemonSetReady(&ds), + wait.WithTimeout(time.Minute*5)) + if err != nil { + t.Fatal(err) + } + return ctx + }). + Assess("MPIJob succeeds", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + rsrc := cfg.Client().Resources() + if err := kubeflowv2beta1.AddToScheme(rsrc.GetScheme()); err != nil { + t.Fatal(err) + } + j := kubeflowv2beta1.MPIJob{ + ObjectMeta: metav1.ObjectMeta{Name: "pytorch-training-multi-node"}, + } + timeout := time.Minute * 20 + err := wait.For(conditions.New(rsrc).ResourceMatch(&j, mpiJobSucceeded), + wait.WithTimeout(timeout)) + if err != nil { + t.Fatal(err) + } + return ctx + }). + Teardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + slices.Reverse(manifestsMultiNode) + err := fwext.DeleteFiles(cfg.Client().RESTConfig(), manifestsMultiNode) + if err != nil { + t.Fatal(err) + } + return ctx + }). + Feature() + + testenv.Test(t, singleNode, multiNode) +} + +func mpiJobSucceeded(obj k8s.Object) bool { + j := obj.(*kubeflowv2beta1.MPIJob) + for _, c := range j.Status.Conditions { + if c.Type == kubeflowv2beta1.JobSucceeded { + return c.Status == v1.ConditionTrue + } + } + return false +}