From 870c7e01edc29d7732e1e157e07c62c033ea2bc5 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Wed, 13 Nov 2024 15:31:30 +0100 Subject: [PATCH 1/6] fix bump-k8s script On-behalf-of: @SAP christoph.mewes@sap.com --- hack/bump-k8s.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/bump-k8s.sh b/hack/bump-k8s.sh index da405eaf3d4..5705dcd0a6c 100755 --- a/hack/bump-k8s.sh +++ b/hack/bump-k8s.sh @@ -29,7 +29,7 @@ set -o xtrace # Note: setting GOPROXY=direct allows us to bump very quickly after the fork has been committed to. GITHUB_USER=${GITHUB_USER:-kcp-dev} GITHUB_REPO=${GITHUB_REPO:-kubernetes} -BRANCH=${BRANCH:-kcp-feature-logical-clusters-1.24-v3} +BRANCH=${BRANCH:-kcp-1.31.0} current_version="$( GOPROXY=direct go mod edit -json | jq '.Replace[] | select(.Old.Path=="k8s.io/kubernetes") | .New.Version' --raw-output )" From 8741fcda4395140f0a711d50abcb5ade1029c30d Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Fri, 29 Nov 2024 19:17:24 +0100 Subject: [PATCH 2/6] add trimmed down version of the built-in k8s authorizer options to the kcp options On-behalf-of: @SAP christoph.mewes@sap.com --- cmd/kcp/kcp.go | 6 +-- pkg/server/config.go | 4 +- pkg/server/options/authorization.go | 77 ++++++++++++++++++++++++++++- pkg/server/options/options.go | 8 +-- test/e2e/framework/kcp.go | 2 +- 5 files changed, 87 insertions(+), 10 deletions(-) diff --git a/cmd/kcp/kcp.go b/cmd/kcp/kcp.go index 56368edc6ad..80b9444829a 100644 --- a/cmd/kcp/kcp.go +++ b/cmd/kcp/kcp.go @@ -117,7 +117,9 @@ func main() { logger := klog.FromContext(cmd.Context()) logger.Info("running with selected batteries", "batteries", strings.Join(completed.Server.Extra.BatteriesIncluded, ",")) - config, err := server.NewConfig(completed.Server) + ctx := genericapiserver.SetupSignalContext() + + serverConfig, err := server.NewConfig(ctx, completed.Server) if err != nil { return err } @@ -127,8 +129,6 @@ func main() { return err } - ctx := genericapiserver.SetupSignalContext() - // the etcd server must be up before NewServer because storage decorators access it right away if completedConfig.EmbeddedEtcd.Config != nil { if err := embeddedetcd.NewServer(completedConfig.EmbeddedEtcd).Run(ctx); err != nil { diff --git a/pkg/server/config.go b/pkg/server/config.go index d03673fe73c..4ce188bcb7c 100644 --- a/pkg/server/config.go +++ b/pkg/server/config.go @@ -175,7 +175,7 @@ func (c *Config) Complete() (CompletedConfig, error) { const KcpBootstrapperUserName = "system:kcp:bootstrapper" -func NewConfig(opts kcpserveroptions.CompletedOptions) (*Config, error) { +func NewConfig(ctx context.Context, opts kcpserveroptions.CompletedOptions) (*Config, error) { c := &Config{ Options: opts, } @@ -324,7 +324,7 @@ func NewConfig(opts kcpserveroptions.CompletedOptions) (*Config, error) { return nil, err } - if err := opts.Authorization.ApplyTo(c.GenericConfig, c.KubeSharedInformerFactory, c.CacheKubeSharedInformerFactory, c.KcpSharedInformerFactory, c.CacheKcpSharedInformerFactory); err != nil { + if err := opts.Authorization.ApplyTo(ctx, c.GenericConfig, c.KubeSharedInformerFactory, c.CacheKubeSharedInformerFactory, c.KcpSharedInformerFactory, c.CacheKcpSharedInformerFactory); err != nil { return nil, err } var userToken string diff --git a/pkg/server/options/authorization.go b/pkg/server/options/authorization.go index 1e06a92d704..532a832d365 100644 --- a/pkg/server/options/authorization.go +++ b/pkg/server/options/authorization.go @@ -17,6 +17,8 @@ limitations under the License. package options import ( + "context" + kcpkubernetesinformers "github.com/kcp-dev/client-go/informers" "github.com/spf13/pflag" @@ -25,7 +27,11 @@ import ( "k8s.io/apiserver/pkg/authorization/authorizerfactory" "k8s.io/apiserver/pkg/authorization/path" "k8s.io/apiserver/pkg/authorization/union" + "k8s.io/apiserver/pkg/informerfactoryhack" genericapiserver "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/server/egressselector" + authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" + kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options" authz "github.com/kcp-dev/kcp/pkg/authorization" kcpinformers "github.com/kcp-dev/kcp/sdk/client/informers/externalversions" @@ -38,6 +44,10 @@ type Authorization struct { // AlwaysAllowGroups are groups which are allowed to take any actions. In kube, this is privileged system group. AlwaysAllowGroups []string + + // Webhook contains flags to enable an external HTTPS webhook to perform + // authorization against. Note that not all built-in options are supported by kcp. + Webhook *kubeoptions.BuiltInAuthorizationOptions } func NewAuthorization() *Authorization { @@ -46,6 +56,7 @@ func NewAuthorization() *Authorization { // This field can be cleared by callers if they don't want this behavior. AlwaysAllowPaths: []string{"/healthz", "/readyz", "/livez"}, AlwaysAllowGroups: []string{user.SystemPrivilegedGroup}, + Webhook: kubeoptions.NewBuiltInAuthorizationOptions(), } } @@ -61,6 +72,22 @@ func (s *Authorization) WithAlwaysAllowPaths(paths ...string) *Authorization { return s } +func (s *Authorization) Complete() error { + if s == nil { + return nil + } + + // kcp only supports optionally specifying an external authorization webhook + // in addition to the built-in authorization logic. + if s.Webhook.WebhookConfigFile != "" { + s.Webhook.Modes = []string{authzmodes.ModeWebhook} + } else { + s.Webhook = nil + } + + return nil +} + func (s *Authorization) Validate() []error { if s == nil { return nil @@ -68,6 +95,12 @@ func (s *Authorization) Validate() []error { allErrors := []error{} + if s.Webhook != nil { + if errs := s.Webhook.Validate(); len(errs) > 0 { + allErrors = append(allErrors, errs...) + } + } + return allErrors } @@ -79,9 +112,23 @@ func (s *Authorization) AddFlags(fs *pflag.FlagSet) { fs.StringSliceVar(&s.AlwaysAllowPaths, "authorization-always-allow-paths", s.AlwaysAllowPaths, "A list of HTTP paths to skip during authorization, i.e. these are authorized without "+ "contacting the 'core' kubernetes server.") + + // Only surface selected, webhook-related CLI flags + + fs.StringVar(&s.Webhook.WebhookConfigFile, "authorization-webhook-config-file", s.Webhook.WebhookConfigFile, + "File with optional webhook configuration in kubeconfig format. The API server will query the remote service to determine access on the API server's secure port.") + + fs.StringVar(&s.Webhook.WebhookVersion, "authorization-webhook-version", s.Webhook.WebhookVersion, + "The API version of the authorization.k8s.io SubjectAccessReview to send to and expect from the webhook.") + + fs.DurationVar(&s.Webhook.WebhookCacheAuthorizedTTL, "authorization-webhook-cache-authorized-ttl", s.Webhook.WebhookCacheAuthorizedTTL, + "The duration to cache 'authorized' responses from the webhook authorizer.") + + fs.DurationVar(&s.Webhook.WebhookCacheUnauthorizedTTL, "authorization-webhook-cache-unauthorized-ttl", s.Webhook.WebhookCacheUnauthorizedTTL, + "The duration to cache 'unauthorized' responses from the webhook authorizer.") } -func (s *Authorization) ApplyTo(config *genericapiserver.Config, kubeInformers, globalKubeInformers kcpkubernetesinformers.SharedInformerFactory, kcpInformers, globalKcpInformers kcpinformers.SharedInformerFactory) error { +func (s *Authorization) ApplyTo(ctx context.Context, config *genericapiserver.Config, kubeInformers, globalKubeInformers kcpkubernetesinformers.SharedInformerFactory, kcpInformers, globalKcpInformers kcpinformers.SharedInformerFactory) error { var authorizers []authorizer.Authorizer localLogicalClusterLister := kcpInformers.Core().V1alpha1().LogicalClusters().Lister() @@ -101,6 +148,34 @@ func (s *Authorization) ApplyTo(config *genericapiserver.Config, kubeInformers, authorizers = append(authorizers, a) } + // Re-use the authorizer from the generic control plane (this is only set for webhooks); + // make sure this is added *after* the alwaysAllow* authorizers, or else the webhook could prevent + // healthcheck endpoints from working. + // NB: Due to the inner workings of Kubernetes' webhook authorizer, this authorizer will actually + // always be a union of a privilegedGroupAuthorizer for system:masters and the webhook itself, + // ensuring the webhook isn't called for that privileged group. + if webhook := s.Webhook; webhook != nil && webhook.WebhookConfigFile != "" { + authorizationConfig, err := webhook.ToAuthorizationConfig(informerfactoryhack.Wrap(kubeInformers)) + if err != nil { + return err + } + + if config.EgressSelector != nil { + egressDialer, err := config.EgressSelector.Lookup(egressselector.ControlPlane.AsNetworkContext()) + if err != nil { + return err + } + authorizationConfig.CustomDial = egressDialer + } + + authorizer, _, err := authorizationConfig.New(ctx, config.APIServerID) + if err != nil { + return err + } + + authorizers = append(authorizers, authorizer) + } + // kcp authorizers, these are evaluated in reverse order // TODO: link the markdown diff --git a/pkg/server/options/options.go b/pkg/server/options/options.go index 2a6a3fc8515..07ecd5b1260 100644 --- a/pkg/server/options/options.go +++ b/pkg/server/options/options.go @@ -29,7 +29,7 @@ import ( genericapiserveroptions "k8s.io/apiserver/pkg/server/options" cliflag "k8s.io/component-base/cli/flag" controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options" - kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options" + authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" kcpadmission "github.com/kcp-dev/kcp/pkg/admission" etcdoptions "github.com/kcp-dev/kcp/pkg/embeddedetcd/options" @@ -251,6 +251,10 @@ func (o *Options) Complete(rootDir string) (*CompletedOptions, error) { o.EmbeddedEtcd.Enabled = true } + if err := o.Authorization.Complete(); err != nil { + return nil, err + } + var err error if !filepath.IsAbs(o.EmbeddedEtcd.Directory) { o.EmbeddedEtcd.Directory, err = filepath.Abs(o.EmbeddedEtcd.Directory) @@ -311,8 +315,6 @@ func (o *Options) Complete(rootDir string) (*CompletedOptions, error) { } if o.GenericControlPlane.ServiceAccountSigningKeyFile == "" { o.GenericControlPlane.ServiceAccountSigningKeyFile = o.Controllers.SAController.ServiceAccountKeyFile - } - completedGenericServerRunOptions, err := o.GenericControlPlane.Complete(nil, nil) if err != nil { return nil, err diff --git a/test/e2e/framework/kcp.go b/test/e2e/framework/kcp.go index 2040af8fb5a..3befdfb64ef 100644 --- a/test/e2e/framework/kcp.go +++ b/test/e2e/framework/kcp.go @@ -655,7 +655,7 @@ func (c *kcpServer) Run(opts ...RunOption) error { return apierrors.NewAggregate(errs) } - config, err := server.NewConfig(completed.Server) + config, err := server.NewConfig(ctx, completed.Server) if err != nil { cleanup() return err From 526cb956ddb7bf05433fcc314f43ed219ee8a3ff Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Fri, 15 Nov 2024 10:51:47 +0100 Subject: [PATCH 3/6] improve variable naming, get rid of misleading TODO On-behalf-of: @SAP christoph.mewes@sap.com --- cmd/kcp/kcp.go | 18 +++++++++--------- pkg/server/options/options.go | 19 +++++-------------- 2 files changed, 14 insertions(+), 23 deletions(-) diff --git a/cmd/kcp/kcp.go b/cmd/kcp/kcp.go index 80b9444829a..e8589991a4c 100644 --- a/cmd/kcp/kcp.go +++ b/cmd/kcp/kcp.go @@ -78,8 +78,8 @@ func main() { } } - serverOptions := options.NewOptions(rootDir) - serverOptions.Server.GenericControlPlane.Logs.Verbosity = logsapiv1.VerbosityLevel(2) + kcpOptions := options.NewOptions(rootDir) + kcpOptions.Server.GenericControlPlane.Logs.Verbosity = logsapiv1.VerbosityLevel(2) startCmd := &cobra.Command{ Use: "start", @@ -101,30 +101,30 @@ func main() { }, RunE: func(cmd *cobra.Command, args []string) error { // run as early as possible to avoid races later when some components (e.g. grpc) start early using klog - if err := logsapiv1.ValidateAndApply(serverOptions.Server.GenericControlPlane.Logs, kcpfeatures.DefaultFeatureGate); err != nil { + if err := logsapiv1.ValidateAndApply(kcpOptions.Server.GenericControlPlane.Logs, kcpfeatures.DefaultFeatureGate); err != nil { return err } - completed, err := serverOptions.Complete() + completedKcpOptions, err := kcpOptions.Complete() if err != nil { return err } - if errs := completed.Validate(); len(errs) > 0 { + if errs := completedKcpOptions.Validate(); len(errs) > 0 { return errors.NewAggregate(errs) } logger := klog.FromContext(cmd.Context()) - logger.Info("running with selected batteries", "batteries", strings.Join(completed.Server.Extra.BatteriesIncluded, ",")) + logger.Info("running with selected batteries", "batteries", strings.Join(completedKcpOptions.Server.Extra.BatteriesIncluded, ",")) ctx := genericapiserver.SetupSignalContext() - serverConfig, err := server.NewConfig(ctx, completed.Server) + serverConfig, err := server.NewConfig(ctx, completedKcpOptions.Server) if err != nil { return err } - completedConfig, err := config.Complete() + completedConfig, err := serverConfig.Complete() if err != nil { return err } @@ -146,7 +146,7 @@ func main() { // add start named flag sets to start flags fss := cliflag.NamedFlagSets{} - serverOptions.AddFlags(&fss) + kcpOptions.AddFlags(&fss) globalflag.AddGlobalFlags(fss.FlagSet("global"), cmd.Name(), logs.SkipLoggingConfigurationFlags()) startFlags := startCmd.Flags() for _, f := range fss.FlagSets { diff --git a/pkg/server/options/options.go b/pkg/server/options/options.go index 07ecd5b1260..f6aafa43e35 100644 --- a/pkg/server/options/options.go +++ b/pkg/server/options/options.go @@ -29,7 +29,6 @@ import ( genericapiserveroptions "k8s.io/apiserver/pkg/server/options" cliflag "k8s.io/component-base/cli/flag" controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options" - authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" kcpadmission "github.com/kcp-dev/kcp/pkg/admission" etcdoptions "github.com/kcp-dev/kcp/pkg/embeddedetcd/options" @@ -114,15 +113,6 @@ func NewOptions(rootDir string) *Options { // override all the stuff o.GenericControlPlane.SecureServing.ServerCert.CertDirectory = rootDir - o.GenericControlPlane.Authentication = kubeoptions.NewBuiltInAuthenticationOptions(). - WithAnonymous(). - WithBootstrapToken(). - WithClientCert(). - WithOIDC(). - WithRequestHeader(). - WithServiceAccounts(). - WithTokenFile(). - WithWebHook() o.GenericControlPlane.Authentication.ServiceAccounts.Issuers = []string{"https://kcp.default.svc"} o.GenericControlPlane.Etcd.StorageConfig.Transport.ServerList = []string{"embedded"} o.GenericControlPlane.Authorization = nil // we have our own @@ -315,7 +305,9 @@ func (o *Options) Complete(rootDir string) (*CompletedOptions, error) { } if o.GenericControlPlane.ServiceAccountSigningKeyFile == "" { o.GenericControlPlane.ServiceAccountSigningKeyFile = o.Controllers.SAController.ServiceAccountKeyFile - completedGenericServerRunOptions, err := o.GenericControlPlane.Complete(nil, nil) + } + + completedGenericOptions, err := o.GenericControlPlane.Complete(nil, nil) if err != nil { return nil, err } @@ -354,7 +346,7 @@ func (o *Options) Complete(rootDir string) (*CompletedOptions, error) { // we already do that for cluster names (stored in the obj) // - we need to modify wildcardClusterNameRegex and crdWildcardPartialMetadataClusterNameRegex o.Cache.Server.Etcd.EnableWatchCache = false - o.Cache.Server.SecureServing = completedGenericServerRunOptions.SecureServing + o.Cache.Server.SecureServing = completedGenericOptions.SecureServing cacheCompletedOptions, err := o.Cache.Complete() if err != nil { return nil, err @@ -362,8 +354,7 @@ func (o *Options) Complete(rootDir string) (*CompletedOptions, error) { return &CompletedOptions{ completedOptions: &completedOptions{ - // TODO: GenericControlPlane here should be completed. But the k/k repo does not expose the CompleteOptions type, but should. - GenericControlPlane: completedGenericServerRunOptions, + GenericControlPlane: completedGenericOptions, EmbeddedEtcd: completedEmbeddedEtcd, Controllers: o.Controllers, Authorization: o.Authorization, From a92ee8c27f375fc267f74935905a5f8e76fda87d Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Thu, 5 Dec 2024 16:28:54 +0100 Subject: [PATCH 4/6] update authorizer documentation On-behalf-of: @SAP christoph.mewes@sap.com --- .../concepts/authorization/authorizers.md | 219 ++++++++++++------ docs/content/concepts/authorization/index.md | 12 +- docs/mkdocs.yml | 7 +- 3 files changed, 162 insertions(+), 76 deletions(-) diff --git a/docs/content/concepts/authorization/authorizers.md b/docs/content/concepts/authorization/authorizers.md index 70a16a83051..cbf69224650 100644 --- a/docs/content/concepts/authorization/authorizers.md +++ b/docs/content/concepts/authorization/authorizers.md @@ -5,46 +5,89 @@ description: > # Authorizers -The following authorizers are configured in kcp: +In kcp, a request has four different ways of being admitted: -| Authorizer | Description | -|----------------------------------------|-----------------------------------------------------------------------------------| -| Top-Level organization authorizer | checks that the user is allowed to access the organization | -| Workspace content authorizer | determines additional groups a user gets inside of a workspace | -| Maximal permission policy authorizer | validates the maximal permission policy RBAC policy in the API exporter workspace | -| Local Policy authorizer | validates the RBAC policy in the workspace that is accessed | -| Kubernetes Bootstrap Policy authorizer | validates the RBAC Kubernetes standard policy | +* It can be made to one of the preconfigured paths that do not require authorization, like `/healthz`. +* It can be performed by a user in one of the configured always-allow groups, by default `system:masters`. +* It can pass through the RBAC chain and match configured Roles and ClusterRoles. +* It can be permitted by an external HTTPS webhook backend. They are related in the following way: -1. top-level organization authorizer must allow -2. workspace content authorizer must allow, and adds additional (virtual per-request) groups to the request user influencing the follow authorizers. -3. maximal permission policy authorizer must allow -4. one of the local authorizer or bootstrap policy authorizer must allow. - -``` - ┌──────────────┐ - │ │ - ┌────►│ Local Policy ├──┐ - ┌──────────────┐ ┌──────────────┐ ┌───────────────────┐ │ │ authorizer │ │ - request │ Workspace │ │ Required │ │ Max. Permission │ │ │ │ │ -─────────►│ Content ├────►│ Groups ├────┤ Policy authorizer ├───┤ └──────────────┘ │ - │ Authorizer │ │ Authorizer │ │ │ │ ▼ - └──────────────┘ └──────────────┘ └───────────────────┘ │ OR───► - │ ┌──────────────┐ ▲ - │ │ Bootstrap │ │ - └────►│ Policy ├──┘ - │ authorizer │ - │ │ - └──────────────┘ +``` mermaid +graph TD + start(Request):::state --> main_alt[/one of\]:::or + main_alt --> aapa[Always Allow Paths Auth] + main_alt --> aaga[Always Allow Groups Auth] + main_alt --> wa[Webhook Auth] + main_alt --> rga[Required Groups Auth] + + aapa --> decision(Decision):::state + aaga --> decision + wa --> decision + + subgraph "RBAC" + rga --> wca[Workspace Content Auth] + wca --> scrda[System CRD Auth] + scrda --> mppa[Max. Permission Policy Auth] + + mppa --- mppa_alt[/one of\]:::or + mppa_alt --> lpa[Local Policy Auth] + mppa_alt --> gpa[Global Policy Auth] + mppa_alt --> bpa[Bootstrap Policy Auth] + end + + lpa --> decision + gpa --> decision + bpa --> decision + + classDef state color:#F77 + classDef or fill:none,stroke:none ``` -[ASCIIFlow document](https://asciiflow.com/#/share/eJyrVspLzE1VslLydg5QcCwtycgvyqxKLVLSUcpJrATSVkrVMUoVMUpWhgYGBjoxSpVAppGlGZBVklpRAuTEKClQGzya0vNoSgPRaEJMTB4N3NCEIUBde9B9OW0XyE6f%2FOTEHIWA%2FJzM5EqgkjnYPfloyh6SENmaSNWDaQQsIEF0Ijx9wSSgoVqUWliaWlwCtk9BITy%2FKLu4IDE5VQEqAKODgMoyi1JTFBASIMo3sUJPISC1KDezuDgzPw8uiWw1ZuRCrMbnflCMAM1xzs8rSc0rwRqGUCXuRfmlBcW44gYWnUjeB0vAI38JVOMUUpL9DMw0CXaLI1Igo4QeFgmYPJbgwQw1hPy0PUM9NWIC%2FyDkrEjtrA5LiKQVbKCg3kQrpwBpp%2Fz8kuKSosQCBZQ8QVXrUNM0pJCDZQioEnghN4NmJXkiStqnsieR7EEToI09pBUTMUq1SrUA%2FWv8Mg%3D%3D) +[View graph on Kroki](https://kroki.io/mermaid/svg/eNqFkkFrwzAMhe_7Faa7dLBux0IOg7ZhvWxQusEOWRmKoyambuTZDln__RQnG02bUp-E3mfx9OzcginEe3wj-DgP1o_X-F2h83dRFHHDo5hMnsQeVPkF2iePVKKg7eeGZbLh2p8WQAADyUzXcHBipjXVYgW-4LryxWYIz0_wpaXKXORrSD4wLYh2lwjLA5sVlMWsPyywjb_AZSiVU1SO4674X7jj8j4XuvVJr42tSvMQ42g9ny1GoWe727Vkw2R3zoBEsaDSY-mPrLMeOCdtBsnbwXnci8U6PkKC1D6C4Wxf4edBrNDulWssiBVpJQ_HKzYY85NQXHy0TguDNc99IQm6P-2My5lbakqvgimDcyLvPAdzzmKZtVa1GQg5H2qmZih6qcG5GLei_amSNNno9nk67atkxVZpHZWcwz17oh2G-he4ue-L) + +### Always Allow Paths Authorizer + +Like in vanilla Kubernetes, this authorizer always grants access to the configured URL paths. This is +used for the health and liveness checks of kcp. + +### Always Allow Groups Authorizer + +This authorizer always permits access if the user is in one of the configured groups. By default this +only includes the `system:masters` group. + +### RBAC Chain + +The primary authorization flow is handled by a sequence of RBAC-based authorizers that a request must +satisfy all in order to be granted access. + +The following authorizers work together to implement RBAC in kcp: +| Authorizer | Description | +|----------------------------------------|--------------------------------------------------------------------------------------------| +| Workspace content authorizer | validates that the user has `access` permission to the workspace | +| Required groups authorizer | validates that the user is in the annotation-based list of groups required for a workspace | +| System CRD authorizer | prevents undesired updates to certain core resources, like the status subresource on APIBindings | +| Maximal permission policy authorizer | validates the maximal permission policy RBAC policy in the API exporter workspace | +| Local Policy authorizer | validates the RBAC policy in the workspace that is accessed | +| Global Policy authorizer | validates the RBAC policy in the workspace that is accessed across shards | +| Kubernetes Bootstrap Policy authorizer | validates the RBAC Kubernetes standard policy | -### Workspace Content Authorizer +#### Required Groups Authorizer -The workspace content authorizer checks whether the user is granted access to the workspace. +A `authorization.kcp.io/required-groups` annotation can be added to a LogicalCluster +to specify additional groups that are required to access a workspace for a user to be member of. +The syntax is a disjunction (separator `,`) of conjunctions (separator `;`). + +For example, `;,` means that a user must be member of `` AND ``, OR of ``. + +The annotation is copied onto sub-workspaces during workspace creation, but is then not updated +automatically if it's changed. + +#### Workspace Content Authorizer + +The workspace content authorizer checks whether the user is granted access to the workspace. Access is granted access through `verb=access` non-resource permission to `/` inside of the workspace. The ClusterRole `system:kcp:workspace:access` is pre-defined which makes it easy @@ -86,39 +129,30 @@ A service-account defined in a workspace implicitly is granted access to it. A service-account defined in a different workspace is NOT given access to it. -### Required Groups Authorizer - -A `authorization.kcp.io/required-groups` annotation can be added to a LogicalCluster -to specify additional groups that are required to access a workspace for a user to be member of. -The syntax is a disjunction (separator `,`) of conjunctions (separator `;`). - -For example, `;,` means that a user must be member of `` AND ``, OR of ``. - -The annotation is copied onto sub-workspaces during scheduling. +!!! note + By default, workspaces are only accessible to a user if they are in `Ready` phase. Workspaces that are initializing + can be accessed only by users that are granted `admin` verb on the `workspaces/content` resource in the + parent workspace. -#### Initializing Workspaces + Service accounts declared within a workspace don't have access to initializing workspaces. -By default, workspaces are only accessible to a user if they are in `Ready` phase. Workspaces that are initializing -can be access only by users that are granted `admin` verb on the `workspaces/content` resource in the -parent workspace. +#### System CRD Authorizer -Service accounts declared within a workspace don't have access to initializing workspaces. +This small authorizer simply prevents updates to the `status` subresource on APIExports or APIBindings. Note that this authorizer does not validate changes to the CustomResourceDefitions themselves, but to objects from those CRDs instead. -### Maximal Permission Policy Authorizer +#### Maximal Permission Policy Authorizer If the requested resource type is part of an API binding, then this authorizer verifies that the request is not exceeding the maximum permission policy of the related API export. Currently, the "local policy" maximum permission policy type is supported. -#### Local Policy +##### Local Policy The local maximum permission policy delegates the decision to the RBAC of the related API export. To distinguish between local RBAC role bindings in that workspace and those for this these maximum permission policy, every name and group is prefixed with `apis.kcp.io:binding:`. -Example: - -Given an API binding for type `foo` declared in workspace `consumer` that refers to an API export declared in workspace `provider` +**Example:** Given an APIBinding for type `foo` declared in workspace `consumer` that refers to an APIExport declared in workspace `provider` and a user `user-1` having the group `group-1` requesting a `create` of `foo` in the `default` namespace in the `consumer` workspace, this authorizer verifies that `user-1` is allowed to execute this request by delegating to `provider`'s RBAC using prefixed attributes. @@ -128,13 +162,13 @@ Using prefixed attributes prevents RBAC collisions i.e. if `user-1` is granted t For the given example RBAC request looks as follows: - Username: `apis.kcp.io:binding:user-1` -- Group: `apis.kcp.io:binding:group-1` +- Groups: [`apis.kcp.io:binding:group-1`] - Resource: `foo` - Namespace: `default` - Workspace: `provider` - Verb: `create` -The following role and role binding declared within the `provider` workspace will grant access to the request: +The following Role and RoleBinding declared within the `provider` workspace will grant access to the request: ```yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -166,32 +200,81 @@ roleRef: ``` !!! note - The same authorization scheme is enforced when executing the request of a claimed resource via the virtual API Export API server, + The same authorization scheme is enforced when executing the request of a claimed resource via the virtual APIExport API server, i.e. a claimed resource is bound to the same maximal permission policy. Only the actual owner of that resources can go beyond that policy. TBD: Example -### Kubernetes Bootstrap Policy Authorizer +#### Local Policy Authorizer -The bootstrap policy authorizer works just like the local authorizer but references RBAC rules -defined in the `system:admin` system workspace. +This authorizer ensures that RBAC rules contained within a workspace are being applied +and work just like in a regular Kubernetes cluster. -### Local Policy Authorizer +It is possible to bind to Roles and ClusterRoles in the bootstrap policy from a local policy's +`RoleBinding` or `ClusterRoleBinding`, for example the `system:kcp:workspace:access` ClusterRole exists in the +`system:admin` logical cluster, but can still be bound from without any other logical cluster. -Once the top-level organization authorizer and the workspace content authorizer granted access to a -workspace, RBAC rules contained in the workspace derived from the request context are evaluated. +#### Global Policy Authorizer -This authorizer ensures that RBAC rules contained within a workspace are being applied -and work just like in a regular Kubernetes cluster. +This authorizer works identically to the Local Policy Authorizer, just with the difference +that it uses a global (i.e. across shards) getter for Roles and RoleBindings. -!!! note - Groups added by the workspace content authorizer can be used for role bindings in that workspace. +#### Bootstrap Policy Authorizer + +The bootstrap policy authorizer works just like the local authorizer but references RBAC rules +defined in the `system:admin` system workspace. This workspace is where the classic Kubernetes +RBAC like the `cluster-admin` ClusterRole is being defined and the policy defined in this workspace +applies to every workspace in a kcp shard. -It is possible to bind to roles and cluster roles in the bootstrap policy from a local policy `RoleBinding` or `ClusterRoleBinding`. +### Webhook Authorizer -### Service Accounts +This authorizer can be enabled by providing the `--authorization-webhook-config-file` flag to the kcp process +and works identically to [how it works in vanilla Kubernetes](https://kubernetes.io/docs/reference/access-authn-authz/webhook/). -Kubernetes service accounts are granted access to the workspaces they are defined in and that are ready. +The given configuration file must be of the kubeconfg format and point to an HTTPS server, potentially including certificate information as needed: -E.g. a service account "default" in `root:org:ws:ws` is granted access to `root:org:ws:ws`, and through the -workspace content authorizer it gains the `system:kcp:clusterworkspace:access` group membership. +```yaml +apiVersion: v1 +kind: Config +clusters: + - name: webhook + cluster: + server: https://localhost:8080/ +current-context: webhook +contexts: + - name: webhook + context: + cluster: webhook +``` + +The webhook will receive every authorization request made in kcp, including internal ones. This means if the webhook +is badly configured, it can even prevent kcp from starting up successfully, but on the other hand this allows +a lot of influence over the authorization in kcp. + +The webhook will receive JSON-marshalled `SubjectAccessReview` objects, that (compared to vanilla Kubernetes) include the name of target logical cluster as an `extra` field, like so: + +```json +{ + "apiVersion": "authorization.k8s.io/v1beta1", + "kind": "SubjectAccessReview", + "spec": { + "resourceAttributes": { + "namespace": "kittensandponies", + "verb": "get", + "group": "unicorn.example.org", + "resource": "pods" + }, + "user": "jane", + "group": [ + "group1", + "group2" + ], + "extra": { + "authorization.kubernetes.io/cluster-name": ["root"] + } + } +} +``` + +!!! note + The extra field will contain the logical cluster _name_ (e.g. o43u2gh528rtfg721rg92), not the human-readable path. Webhooks need to resolve the name to a path themselves if necessary. diff --git a/docs/content/concepts/authorization/index.md b/docs/content/concepts/authorization/index.md index a62ee1f73f7..507334551c8 100644 --- a/docs/content/concepts/authorization/index.md +++ b/docs/content/concepts/authorization/index.md @@ -11,13 +11,11 @@ Generally, the same (cluster) role and (cluster) role binding principles apply e In addition, additional RBAC semantics is implemented cross-workspaces, namely the following: -- **Top-Level Organization** access: the user must have this as pre-requisite to access any other workspace, or is - even member and by that can create workspaces inside the organization workspace. -- **Workspace Content** access: the user needs access to a workspace or is even admin. -- for some resources, additional permission checks are performed, not represented by local or Kubernetes standard RBAC rules. E.g. - - workspace creation checks for organization membership (see above). - - workspace creation checks for `use` verb on the `WorkspaceType`. - - API binding via APIBinding objects requires verb `bind` access to the corresponding `APIExport`. +- **Workspace Content** access: the user needs `access` permissions to a workspace or be even admin. +- for some resources, additional permission checks are performed, not represented by local or Kubernetes standard RBAC rules; for example + - workspace creation checks for organization membership (see above). + - workspace creation checks for `use` verb on the `WorkspaceType`. + - API binding via APIBinding objects requires verb `bind` access to the corresponding `APIExport`. - **System Workspaces** access: system workspaces are prefixed with `system:` and are not accessible by users. The details of the authorizer chain are documented in [Authorizers](./authorizers.md). diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 835bfebdbb7..08fa339bc99 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -37,7 +37,7 @@ theme: # Palette toggle for light mode - media: "(prefers-color-scheme: light)" - scheme: default + scheme: default primary: white toggle: icon: material/brightness-7 @@ -92,6 +92,11 @@ markdown_extensions: - pymdownx.highlight: # Allows linking directly to specific lines in code blocks anchor_linenums: true + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format # Inline code block highlighting - pymdownx.inlinehilite # Lets you embed content from another file From c96efe11da0047f60b28171a655170fbbcc3a6f7 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Fri, 13 Dec 2024 13:16:51 +0100 Subject: [PATCH 5/6] add very basic tests for the webhook functionality On-behalf-of: @SAP christoph.mewes@sap.com --- test/e2e/authorizer/webhook.kubeconfig | 12 +++ test/e2e/authorizer/webhook_test.go | 121 +++++++++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 test/e2e/authorizer/webhook.kubeconfig create mode 100644 test/e2e/authorizer/webhook_test.go diff --git a/test/e2e/authorizer/webhook.kubeconfig b/test/e2e/authorizer/webhook.kubeconfig new file mode 100644 index 00000000000..5d783969d50 --- /dev/null +++ b/test/e2e/authorizer/webhook.kubeconfig @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Config +clusters: + - name: httest + cluster: + certificate-authority: .httest/ca.crt + server: https://localhost:8080/ +current-context: webhook +contexts: + - name: webhook + context: + cluster: httest diff --git a/test/e2e/authorizer/webhook_test.go b/test/e2e/authorizer/webhook_test.go new file mode 100644 index 00000000000..05df5d9df81 --- /dev/null +++ b/test/e2e/authorizer/webhook_test.go @@ -0,0 +1,121 @@ +/* +Copyright 2024 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package authorizer + +import ( + "context" + "os/exec" + "testing" + "time" + + kcpkubernetesclientset "github.com/kcp-dev/client-go/kubernetes" + "github.com/kcp-dev/logicalcluster/v3" + "github.com/stretchr/testify/require" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubernetesscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + + kcpclientset "github.com/kcp-dev/kcp/sdk/client/clientset/versioned/cluster" + "github.com/kcp-dev/kcp/test/e2e/framework" +) + +func TestWebhook(t *testing.T) { + framework.Suite(t, "control-plane") + + ctx, cancelFunc := context.WithCancel(context.Background()) + t.Cleanup(cancelFunc) + + // start a webhook that allows kcp to boot up + webhookStop := runWebhook(ctx, t, "kubernetes:authz:allow") + t.Cleanup(webhookStop) + + server := framework.PrivateKcpServer(t, framework.WithCustomArguments( + "--authorization-webhook-config-file", + "webhook.kubeconfig", + )) + + // create clients + kcpConfig := server.BaseConfig(t) + kubeClusterClient, err := kcpkubernetesclientset.NewForConfig(kcpConfig) + require.NoError(t, err, "failed to construct client for server") + kcpClusterClient, err := kcpclientset.NewForConfig(kcpConfig) + require.NoError(t, err, "failed to construct client for server") + + t.Log("Admin should be allowed to list Workspaces.") + _, err = kcpClusterClient.Cluster(logicalcluster.NewPath("root")).TenancyV1alpha1().Workspaces().List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + + // stop the webhook and switch to a deny policy + webhookStop() + + webhookStop = runWebhook(ctx, t, "kubernetes:authz:deny") + t.Cleanup(webhookStop) + + t.Log("Admin should not be allowed to list ConfigMaps.") + _, err = kubeClusterClient.Cluster(logicalcluster.NewPath("root")).CoreV1().ConfigMaps("default").List(ctx, metav1.ListOptions{}) + require.Error(t, err) + + // access to health endpoints should still be granted based on --always-allow-paths, + // even if the webhook rejects the request + rootShardCfg := server.RootShardSystemMasterBaseConfig(t) + if rootShardCfg.NegotiatedSerializer == nil { + rootShardCfg.NegotiatedSerializer = kubernetesscheme.Codecs.WithoutConversion() + } + + // Ensure the request is unauthenticated, as Kubernetes' webhook authorizer is wrapped + // in a reloadable authorizer that also always injects a privilegedGroup authorizer + // that lets system:masters users in. + rootShardCfg.BearerToken = "" + + restClient, err := rest.UnversionedRESTClientFor(rootShardCfg) + require.NoError(t, err) + + for _, endpoint := range []string{"/livez", "/readyz"} { + req := rest.NewRequest(restClient).RequestURI(endpoint) + t.Logf("%s should still be accessible.", req.URL().String()) + _, err := req.Do(ctx).Raw() + require.NoError(t, err) + } +} + +func runWebhook(ctx context.Context, t *testing.T, response string) context.CancelFunc { + args := []string{ + "--tls", + "--response", response, + } + + t.Logf("Starting webhook with %s policy...", response) + + ctx, cancel := context.WithCancel(ctx) + + cmd := exec.CommandContext(ctx, "httest", args...) + if err := cmd.Start(); err != nil { + cancel() + t.Fatalf("Failed to start webhook: %v", err) + } + + // give httest a moment to boot up + time.Sleep(2 * time.Second) + + return func() { + t.Log("Stopping webhook...") + cancel() + // give it some time to shutdown + time.Sleep(2 * time.Second) + } +} From f99e7557815860821282f8b018b2fef119d0af9d Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Fri, 13 Dec 2024 13:00:33 +0100 Subject: [PATCH 6/6] add httest installation to the Makefile On-behalf-of: @SAP christoph.mewes@sap.com --- Makefile | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ce8f7eafc6e..721d4268ac1 100644 --- a/Makefile +++ b/Makefile @@ -59,6 +59,10 @@ GOLANGCI_LINT_VER := v1.62.2 GOLANGCI_LINT_BIN := golangci-lint GOLANGCI_LINT := $(TOOLS_GOBIN_DIR)/$(GOLANGCI_LINT_BIN)-$(GOLANGCI_LINT_VER) +HTTEST_VER := v0.3.2 +HTTEST_BIN := httest +HTTEST := $(TOOLS_GOBIN_DIR)/$(HTTEST_BIN)-$(HTTEST_VER) + GOTESTSUM_VER := v1.8.1 GOTESTSUM_BIN := gotestsum GOTESTSUM := $(abspath $(TOOLS_DIR))/$(GOTESTSUM_BIN)-$(GOTESTSUM_VER) @@ -136,6 +140,9 @@ install: require-jq require-go require-git verify-go-versions ## Install the pro $(GOLANGCI_LINT): GOBIN=$(TOOLS_GOBIN_DIR) $(GO_INSTALL) github.com/golangci/golangci-lint/cmd/golangci-lint $(GOLANGCI_LINT_BIN) $(GOLANGCI_LINT_VER) +$(HTTEST): + GOBIN=$(TOOLS_GOBIN_DIR) $(GO_INSTALL) go.xrstf.de/httest $(HTTEST_BIN) $(HTTEST_VER) + $(LOGCHECK): GOBIN=$(TOOLS_GOBIN_DIR) $(GO_INSTALL) sigs.k8s.io/logtools/logcheck $(LOGCHECK_BIN) $(LOGCHECK_VER) @@ -183,7 +190,7 @@ vendor: ## Vendor the dependencies go mod vendor .PHONY: vendor -tools: $(GOLANGCI_LINT) $(CONTROLLER_GEN) $(KCP_APIGEN_GEN) $(YAML_PATCH) $(GOTESTSUM) $(OPENSHIFT_GOIMPORTS) $(CODE_GENERATOR) ## Install tools +tools: $(GOLANGCI_LINT) $(HTTEST) $(CONTROLLER_GEN) $(KCP_APIGEN_GEN) $(YAML_PATCH) $(GOTESTSUM) $(OPENSHIFT_GOIMPORTS) $(CODE_GENERATOR) ## Install tools .PHONY: tools $(CONTROLLER_GEN): @@ -269,6 +276,7 @@ endif ifdef USE_GOTESTSUM test-e2e: $(GOTESTSUM) endif +test-e2e: $(HTTEST) test-e2e: TEST_ARGS ?= test-e2e: WHAT ?= ./test/e2e... test-e2e: build-all ## Run e2e tests @@ -280,6 +288,7 @@ test-e2e: build-all ## Run e2e tests ifdef USE_GOTESTSUM test-e2e-shared-minimal: $(GOTESTSUM) endif +test-e2e-shared-minimal: $(HTTEST) test-e2e-shared-minimal: TEST_ARGS ?= test-e2e-shared-minimal: WHAT ?= ./test/e2e... test-e2e-shared-minimal: WORK_DIR ?= . @@ -305,6 +314,7 @@ test-e2e-shared-minimal: build-all ifdef USE_GOTESTSUM test-e2e-sharded-minimal: $(GOTESTSUM) endif +test-e2e-sharded-minimal: $(HTTEST) test-e2e-sharded-minimal: TEST_ARGS ?= test-e2e-sharded-minimal: WHAT ?= ./test/e2e... test-e2e-sharded-minimal: WORK_DIR ?= .