Add better SA auth handling

On-behalf-of: SAP <[email protected]>
Signed-off-by: Mangirdas Judeikis <[email protected]>

Author: mjudeikis

DEVELOPMENT.md

@@ -14,3 +14,60 @@ The project has been initialised with:
 ```sh
 $ kubebuilder init --domain operator.kcp.io --owner "The KCP Authors" --project-name kcp-operator
 ```
+
+
+## Kind setup
+
+```sh
+kind create cluster
+```
+
+Install cert-manager:
+
+```sh
+kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml
+```
+
+Install etcd:
+
+```sh
+helm install etcd oci://registry-1.docker.io/bitnamicharts/etcd --set auth.rbac.enabled=false --set auth.rbac.create=false
+helm install etcd-shard oci://registry-1.docker.io/bitnamicharts/etcd --set auth.rbac.enabled=false --set auth.rbac.create=false
+```
+
+Create issuer:
+
+```sh
+kubectl apply -f config/samples/cert-manager/issuer.yaml
+```
+
+Build the image:
+
+```sh
+make docker-build IMG=ghcr.io/kcp-dev/kcp-operator:1
+kind load docker-image ghcr.io/kcp-dev/kcp-operator:1
+```
+
+Load the image into the kind cluster:
+
+```sh
+make deploy IMG=ghcr.io/kcp-dev/kcp-operator:1
+```
+
+Now you can create a root shard:
+
+```sh
+kubectl apply -f config/samples/operator.kcp.io_v1alpha1_rootshard.yaml       
+```
+
+Shards:
+
+```sh
+kubectl apply -f config/samples/operator.kcp.io_v1alpha1_shard.yaml
+```
+
+Frontproxy:
+
+```sh
+kubectl apply -f config/samples/operator.kcp.io_v1alpha1_frontproxy.yaml
+```

config/crd/bases/operator.kcp.io_frontproxies.yaml

@@ -81,8 +81,9 @@ spec:
                   and Authorization for this front-proxy instance.'
                 properties:
                   dropGroups:
-                    description: 'Optional: DropGroups configures groups to be dropped
-                      before forwarding requests to Shards'
+                    description: |-
+                      Optional: DropGroups configures groups to be dropped before forwarding requests to Shards.
+                      If set, it will mount every shard's service account certificate to the front-proxy.
                     items:
                       type: string
                     type: array
@@ -131,6 +132,17 @@ spec:
                     items:
                       type: string
                     type: array
+                  serviceAccount:
+                    description: 'Optional: serviceAccountAuthentication configures
+                      ServiceAccount Authentication.'
+                    properties:
+                      enabled:
+                        description: 'Optional: Enabled enables or disables ServiceAccount
+                          Authentication.'
+                        type: boolean
+                    required:
+                    - enabled
+                    type: object
                 type: object
               certificateTemplates:
                 additionalProperties:

config/crd/bases/operator.kcp.io_rootshards.yaml

@@ -120,6 +120,74 @@ spec:
                         type: string
                     type: object
                 type: object
+              auth:
+                description: 'Optional: Auth configures various aspects of Authentication
+                  and Authorization for this shard.'
+                properties:
+                  dropGroups:
+                    description: |-
+                      Optional: DropGroups configures groups to be dropped before forwarding requests to Shards.
+                      If set, it will mount every shard's service account certificate to the front-proxy.
+                    items:
+                      type: string
+                    type: array
+                  oidc:
+                    description: 'Optional: OIDC configures OpenID Connect Authentication.'
+                    properties:
+                      clientID:
+                        description: ClientID is the OIDC client ID configured on
+                          the issuer side for this KCP instance.
+                        type: string
+                      clientSecret:
+                        description: |-
+                          Optionally provide the client secret for the OIDC client. This is not used by KCP itself, but is used to generate
+                          a OIDC kubeconfig that can be shared with users to log in via the OIDC provider.
+                        type: string
+                      groupsClaim:
+                        description: 'Experimental: Optionally provides a custom claim
+                          for fetching groups. The claim must be a string or an array
+                          of strings.'
+                        type: string
+                      groupsPrefix:
+                        description: |-
+                          Optionally sets a custom groups prefix. This defaults to "oidc:" if unset, which means a group called "group1"
+                          on the OIDC side will be recognised as "oidc:group1" in KCP.
+                        type: string
+                      issuerURL:
+                        description: IssuerURL is used for the OIDC issuer URL. Only
+                          https URLs will be accepted.
+                        type: string
+                      usernameClaim:
+                        description: Optionally uses a custom claim for fetching the
+                          username. This defaults to "sub" if unset.
+                        type: string
+                      usernamePrefix:
+                        description: |-
+                          Optionally sets a custom username prefix. This defaults to "oidc:" if unset, which means a user called "[email protected]"
+                          on the OIDC side will be recognised as "oidc:[email protected]" in KCP.
+                        type: string
+                    required:
+                    - clientID
+                    - issuerURL
+                    type: object
+                  passOnGroups:
+                    description: 'Optional: PassOnGroups configures groups to be passed
+                      on before forwarding requests to Shards'
+                    items:
+                      type: string
+                    type: array
+                  serviceAccount:
+                    description: 'Optional: serviceAccountAuthentication configures
+                      ServiceAccount Authentication.'
+                    properties:
+                      enabled:
+                        description: 'Optional: Enabled enables or disables ServiceAccount
+                          Authentication.'
+                        type: boolean
+                    required:
+                    - enabled
+                    type: object
+                type: object
               authorization:
                 properties:
                   webhook:
@@ -1742,6 +1810,21 @@ spec:
                 x-kubernetes-list-type: map
               phase:
                 type: string
+              shards:
+                description: Shards is a list of shards that are currently registered
+                  with this root shard.
+                items:
+                  properties:
+                    name:
+                      description: Name is the name of the shard.
+                      type: string
+                  required:
+                  - name
+                  type: object
+                type: array
+                x-kubernetes-list-map-keys:
+                - name
+                x-kubernetes-list-type: map
             type: object
         type: object
     served: true

config/crd/bases/operator.kcp.io_shards.yaml

@@ -120,6 +120,74 @@ spec:
                         type: string
                     type: object
                 type: object
+              auth:
+                description: 'Optional: Auth configures various aspects of Authentication
+                  and Authorization for this shard.'
+                properties:
+                  dropGroups:
+                    description: |-
+                      Optional: DropGroups configures groups to be dropped before forwarding requests to Shards.
+                      If set, it will mount every shard's service account certificate to the front-proxy.
+                    items:
+                      type: string
+                    type: array
+                  oidc:
+                    description: 'Optional: OIDC configures OpenID Connect Authentication.'
+                    properties:
+                      clientID:
+                        description: ClientID is the OIDC client ID configured on
+                          the issuer side for this KCP instance.
+                        type: string
+                      clientSecret:
+                        description: |-
+                          Optionally provide the client secret for the OIDC client. This is not used by KCP itself, but is used to generate
+                          a OIDC kubeconfig that can be shared with users to log in via the OIDC provider.
+                        type: string
+                      groupsClaim:
+                        description: 'Experimental: Optionally provides a custom claim
+                          for fetching groups. The claim must be a string or an array
+                          of strings.'
+                        type: string
+                      groupsPrefix:
+                        description: |-
+                          Optionally sets a custom groups prefix. This defaults to "oidc:" if unset, which means a group called "group1"
+                          on the OIDC side will be recognised as "oidc:group1" in KCP.
+                        type: string
+                      issuerURL:
+                        description: IssuerURL is used for the OIDC issuer URL. Only
+                          https URLs will be accepted.
+                        type: string
+                      usernameClaim:
+                        description: Optionally uses a custom claim for fetching the
+                          username. This defaults to "sub" if unset.
+                        type: string
+                      usernamePrefix:
+                        description: |-
+                          Optionally sets a custom username prefix. This defaults to "oidc:" if unset, which means a user called "[email protected]"
+                          on the OIDC side will be recognised as "oidc:[email protected]" in KCP.
+                        type: string
+                    required:
+                    - clientID
+                    - issuerURL
+                    type: object
+                  passOnGroups:
+                    description: 'Optional: PassOnGroups configures groups to be passed
+                      on before forwarding requests to Shards'
+                    items:
+                      type: string
+                    type: array
+                  serviceAccount:
+                    description: 'Optional: serviceAccountAuthentication configures
+                      ServiceAccount Authentication.'
+                    properties:
+                      enabled:
+                        description: 'Optional: Enabled enables or disables ServiceAccount
+                          Authentication.'
+                        type: boolean
+                    required:
+                    - enabled
+                    type: object
+                type: object
               authorization:
                 properties:
                   webhook:

config/samples/v1alpha1_kubeconfig_admin.yaml

@@ -0,0 +1,17 @@
+apiVersion: operator.kcp.io/v1alpha1
+kind: Kubeconfig
+metadata:
+  labels:
+    app.kubernetes.io/name: kcp-operator
+    app.kubernetes.io/managed-by: kustomize
+  name: kubeconfig-admin-sample
+spec:
+  username: [email protected]
+  groups:
+    - kcp-admin
+  validity: 8766h
+  secretRef:
+    name: sample-admin-kubeconfig
+  target:
+    frontProxyRef:
+      name: frontproxy-sample

internal/controller/rootshard_controller.go

@@ -19,6 +19,7 @@ package controller
 import (
 	"context"
 	"fmt"
+	"sort"
 
 	certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
 	k8creconciling "k8c.io/reconciler/pkg/reconciling"
@@ -31,9 +32,12 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	kerrors "k8s.io/apimachinery/pkg/util/errors"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
 	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	"github.com/kcp-dev/kcp-operator/internal/reconciling"
 	"github.com/kcp-dev/kcp-operator/internal/resources"
@@ -49,13 +53,29 @@ type RootShardReconciler struct {
 
 // SetupWithManager sets up the controller with the Manager.
 func (r *RootShardReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	shardHandler := handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, obj ctrlruntimeclient.Object) []reconcile.Request {
+		shard := obj.(*operatorv1alpha1.Shard)
+
+		var rootShard operatorv1alpha1.RootShard
+		if err := mgr.GetClient().Get(ctx, ctrlruntimeclient.ObjectKey{Namespace: shard.Namespace, Name: shard.Spec.RootShard.Reference.Name}, &rootShard); err != nil {
+			utilruntime.HandleError(err)
+			return nil
+		}
+
+		var requests []reconcile.Request
+		requests = append(requests, reconcile.Request{NamespacedName: ctrlruntimeclient.ObjectKeyFromObject(&rootShard)})
+
+		return requests
+	})
+
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&operatorv1alpha1.RootShard{}).
 		Owns(&appsv1.Deployment{}).
 		Owns(&corev1.ConfigMap{}).
 		Owns(&corev1.Service{}).
 		Owns(&corev1.Secret{}).
 		Owns(&certmanagerv1.Certificate{}).
+		Watches(&operatorv1alpha1.Shard{}, shardHandler).
 		Complete(r)
 }
 
@@ -197,6 +217,20 @@ func (r *RootShardReconciler) reconcileStatus(ctx context.Context, oldRootShard
 		rootShard.Status.Phase = operatorv1alpha1.RootShardPhaseProvisioning
 	}
 
+	shards, err := getRootShardChildren(ctx, r.Client, rootShard)
+	if err != nil {
+		errs = append(errs, err)
+	} else {
+		rootShard.Status.Shards = make([]operatorv1alpha1.ShardReference, len(shards))
+		for i, shard := range shards {
+			rootShard.Status.Shards[i] = operatorv1alpha1.ShardReference{Name: shard.Name}
+		}
+	}
+	// sort the shards by name for equality comparison
+	sort.Slice(rootShard.Status.Shards, func(i, j int) bool {
+		return rootShard.Status.Shards[i].Name < rootShard.Status.Shards[j].Name
+	})
+
 	// only patch the status if there are actual changes.
 	if !equality.Semantic.DeepEqual(oldRootShard.Status, rootShard.Status) {
 		if err := r.Status().Patch(ctx, rootShard, ctrlruntimeclient.MergeFrom(oldRootShard)); err != nil {

internal/controller/util.go

@@ -25,6 +25,7 @@ import (
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	kerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/utils/ptr"
 	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
 
@@ -117,3 +118,25 @@ func fetchRootShard(ctx context.Context, client ctrlruntimeclient.Client, namesp
 		Message: "RootShard reference is valid.",
 	}, rootShard
 }
+
+// getRootShardChildren returns all shards that are currently registered with the given root shard.
+func getRootShardChildren(ctx context.Context, client ctrlruntimeclient.Client, rootShard *operatorv1alpha1.RootShard) ([]operatorv1alpha1.Shard, error) {
+	var errs []error
+
+	var shards operatorv1alpha1.ShardList
+	err := client.List(ctx, &shards, &ctrlruntimeclient.ListOptions{
+		Namespace: rootShard.Namespace,
+	})
+	if err != nil {
+		errs = append(errs, err)
+	}
+
+	var result []operatorv1alpha1.Shard
+	for _, shard := range shards.Items {
+		if shard.Spec.RootShard.Reference.Name == rootShard.Name {
+			result = append(result, shard)
+		}
+	}
+
+	return result, kerrors.NewAggregate(errs)
+}