From f002f6928d04b0305840f321641bf1359eb98fa2 Mon Sep 17 00:00:00 2001 From: Mohamed Mahmoud Date: Thu, 23 May 2024 15:34:36 -0400 Subject: [PATCH 01/16] WIP: introducing bfpapplication object Signed-off-by: Mohamed Mahmoud --- PROJECT | 18 +- apis/v1alpha1/bpfapplication_types.go | 150 ++ apis/v1alpha1/zz_generated.deepcopy.go | 158 ++ apis/v1alpha1/zz_generated.register.go | 2 + ...bpfman-operator.clusterserviceversion.yaml | 1668 +++++++---------- cmd/bpfman-operator/main.go | 6 + .../crd/bases/bpfman.io_bpfapplications.yaml | 1548 +++++++++++++++ config/crd/kustomization.yaml | 4 +- .../cainjection_in_bpfapplications.yaml | 7 + .../patches/webhook_in_bpfapplications.yaml | 16 + config/rbac/bpfapplication_editor_role.yaml | 31 + config/rbac/bpfapplication_viewer_role.yaml | 27 + config/rbac/bpfman-operator/role.yaml | 26 + config/samples/_v1alpha1_bpfapplication.yaml | 12 + config/samples/kustomization.yaml | 1 + .../bpfman-operator/application-programs.go | 162 ++ internal/constants.go | 2 + pkg/client/apis/v1alpha1/bpfapplication.go | 68 + .../apis/v1alpha1/expansion_generated.go | 4 + .../typed/apis/v1alpha1/apis_client.go | 5 + .../typed/apis/v1alpha1/bpfapplication.go | 184 ++ .../apis/v1alpha1/fake/fake_apis_client.go | 4 + .../apis/v1alpha1/fake/fake_bpfapplication.go | 132 ++ .../apis/v1alpha1/generated_expansion.go | 2 + .../apis/v1alpha1/bpfapplication.go | 89 + .../apis/v1alpha1/interface.go | 7 + pkg/client/externalversions/generic.go | 2 + pkg/helpers/helpers.go | 10 +- 28 files changed, 3370 insertions(+), 975 deletions(-) create mode 100644 apis/v1alpha1/bpfapplication_types.go create mode 100644 config/crd/bases/bpfman.io_bpfapplications.yaml create mode 100644 config/crd/patches/cainjection_in_bpfapplications.yaml create mode 100644 config/crd/patches/webhook_in_bpfapplications.yaml create mode 100644 config/rbac/bpfapplication_editor_role.yaml create mode 100644 config/rbac/bpfapplication_viewer_role.yaml create mode 100644 config/samples/_v1alpha1_bpfapplication.yaml create mode 100644 controllers/bpfman-operator/application-programs.go create mode 100644 pkg/client/apis/v1alpha1/bpfapplication.go create mode 100644 pkg/client/clientset/typed/apis/v1alpha1/bpfapplication.go create mode 100644 pkg/client/clientset/typed/apis/v1alpha1/fake/fake_bpfapplication.go create mode 100644 pkg/client/externalversions/apis/v1alpha1/bpfapplication.go diff --git a/PROJECT b/PROJECT index 89787a1ad..7ee64d607 100644 --- a/PROJECT +++ b/PROJECT @@ -1,3 +1,7 @@ +# Code generated by tool. DO NOT EDIT. +# This file is used to track the info used to scaffold your project +# and allow the plugins properly work. +# More info: https://book.kubebuilder.io/reference/project-config.html domain: bpfman.io layout: - go.kubebuilder.io/v3 @@ -17,7 +21,6 @@ resources: version: v1alpha1 - api: crdVersion: v1 - namespaced: false controller: true domain: bpfman.io kind: XdpProgram @@ -25,7 +28,6 @@ resources: version: v1alpha1 - api: crdVersion: v1 - namespaced: false controller: true domain: bpfman.io kind: TcProgram @@ -33,7 +35,6 @@ resources: version: v1alpha1 - api: crdVersion: v1 - namespaced: false controller: true domain: bpfman.io kind: TracePointProgram @@ -41,7 +42,6 @@ resources: version: v1alpha1 - api: crdVersion: v1 - namespaced: false controller: true domain: bpfman.io kind: KprobeProgram @@ -49,7 +49,6 @@ resources: version: v1alpha1 - api: crdVersion: v1 - namespaced: false controller: true domain: bpfman.io kind: UprobeProgram @@ -57,7 +56,6 @@ resources: version: v1alpha1 - api: crdVersion: v1 - namespaced: false controller: true domain: bpfman.io kind: FentryProgram @@ -65,10 +63,16 @@ resources: version: v1alpha1 - api: crdVersion: v1 - namespaced: false controller: true domain: bpfman.io kind: FexitProgram path: github.com/bpfman/bpfman-operator/apis/v1alpha1 version: v1alpha1 +- api: + crdVersion: v1 + controller: true + domain: bpfman.io + kind: BpfApplication + path: github.com/bpfman/api/v1alpha1 + version: v1alpha1 version: "3" diff --git a/apis/v1alpha1/bpfapplication_types.go b/apis/v1alpha1/bpfapplication_types.go new file mode 100644 index 000000000..40d11c889 --- /dev/null +++ b/apis/v1alpha1/bpfapplication_types.go @@ -0,0 +1,150 @@ +/* +Copyright 2023 The bpfman Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EBPFProgType defines the supported eBPF program types +type EBPFProgType string + +const ( + // ProgTypeXDP refers to the eBPF XDP programs type. + ProgTypeXDP EBPFProgType = "XDP" + + // ProgTypeTC refers to the eBPF TC programs type. + ProgTypeTC EBPFProgType = "TC" + + // ProgTypeTCX refers to the eBPF TCx programs type. + ProgTypeTCX EBPFProgType = "TCX" + + // ProgTypeFentry refers to the eBPF Fentry programs type. + ProgTypeFentry EBPFProgType = "Fentry" + + // ProgTypeFexit refers to the eBPF Fexit programs type. + ProgTypeFexit EBPFProgType = "Fexit" + + // ProgTypeKprobe refers to the eBPF Kprobe programs type. + ProgTypeKprobe EBPFProgType = "Kprobe" + + // ProgTypeKretprobe refers to the eBPF Kprobe programs type. + ProgTypeKretprobe EBPFProgType = "Kretprobe" + + // ProgTypeUprobe refers to the eBPF Uprobe programs type. + ProgTypeUprobe EBPFProgType = "Uprobe" + + // ProgTypeUretprobe refers to the eBPF Uretprobe programs type. + ProgTypeUretprobe EBPFProgType = "Uretprobe" + + // ProgTypeTracepoint refers to the eBPF Tracepoint programs type. + ProgTypeTracepoint EBPFProgType = "Tracepoint" +) + +// BpfApplicationProgram defines the desired state of BpfApplication +type BpfApplicationProgram struct { + // Type specifies the bpf program type + // +unionDiscriminator + // +kubebuilder:validation:Required + // +kubebuilder:validation:Enum:="XDP";"TC";"TCX";"Fentry";"Fexit";"Kprobe";"Kretprobe";"Uprobe";"Uretprobe";"Tracepoint" + // +optional + Type EBPFProgType `json:"type,omitempty"` + + // xdp defines the desired state of the application's XdpPrograms. + // +unionMember + // +optional + XDP *XdpProgramInfo `json:"xdp,omitempty"` + + // tc defines the desired state of the application's TcPrograms. + // +unionMember + // +optional + TC *TcProgramInfo `json:"tc,omitempty"` + + // fentry defines the desired state of the application's FentryPrograms. + // +unionMember + // +optional + Fentry *FentryProgramInfo `json:"fentry,omitempty"` + + // fexit defines the desired state of the application's FexitPrograms. + // +unionMember + // +optional + Fexit *FexitProgramInfo `json:"fexit,omitempty"` + + // kprobe defines the desired state of the application's KprobePrograms. + // +unionMember + // +optional + Kprobe *KprobeProgramInfo `json:"kprobe,omitempty"` + + // kretprobe defines the desired state of the application's KretprobePrograms. + // +unionMember + // +optional + Kretprobe *KprobeProgramInfo `json:"kretprobe,omitempty"` + + // uprobe defines the desired state of the application's UprobePrograms. + // +unionMember + // +optional + Uprobe *UprobeProgramInfo `json:"uprobe,omitempty"` + + // uretprobe defines the desired state of the application's UretprobePrograms. + // +unionMember + // +optional + Uretprobe *UprobeProgramInfo `json:"uretprobe,omitempty"` + + // tracepoint defines the desired state of the application's TracepointPrograms. + // +unionMember + // +optional + Tracepoint *TracepointProgramInfo `json:"tracepoint,omitempty"` +} + +// BpfApplicationSpec defines the desired state of BpfApplication +type BpfApplicationSpec struct { + BpfAppCommon `json:",inline"` + + // Programs is a list of bpf programs supported for a specific application. + // It's possible that the application can selectively choose which program(s) + // to run from this list. + // +kubebuilder:validation:MinItems:=1 + Programs []BpfApplicationProgram `json:"programs,omitempty"` +} + +// BpfApplicationStatus defines the observed state of BpfApplication +type BpfApplicationStatus struct { + BpfProgramStatusCommon `json:",inline"` +} + +// +genclient +// +genclient:nonNamespaced +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:resource:scope=Cluster + +// BpfApplication is the Schema for the bpfapplications API +type BpfApplication struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec BpfApplicationSpec `json:"spec,omitempty"` + Status BpfApplicationStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true +// BpfApplicationList contains a list of BpfApplication +type BpfApplicationList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []BpfApplication `json:"items"` +} diff --git a/apis/v1alpha1/zz_generated.deepcopy.go b/apis/v1alpha1/zz_generated.deepcopy.go index 303fb614e..ca038cf73 100644 --- a/apis/v1alpha1/zz_generated.deepcopy.go +++ b/apis/v1alpha1/zz_generated.deepcopy.go @@ -58,6 +58,164 @@ func (in *BpfAppCommon) DeepCopy() *BpfAppCommon { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BpfApplication) DeepCopyInto(out *BpfApplication) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BpfApplication. +func (in *BpfApplication) DeepCopy() *BpfApplication { + if in == nil { + return nil + } + out := new(BpfApplication) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BpfApplication) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BpfApplicationList) DeepCopyInto(out *BpfApplicationList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]BpfApplication, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BpfApplicationList. +func (in *BpfApplicationList) DeepCopy() *BpfApplicationList { + if in == nil { + return nil + } + out := new(BpfApplicationList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BpfApplicationList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BpfApplicationProgram) DeepCopyInto(out *BpfApplicationProgram) { + *out = *in + if in.XDP != nil { + in, out := &in.XDP, &out.XDP + *out = new(XdpProgramInfo) + (*in).DeepCopyInto(*out) + } + if in.TC != nil { + in, out := &in.TC, &out.TC + *out = new(TcProgramInfo) + (*in).DeepCopyInto(*out) + } + if in.Fentry != nil { + in, out := &in.Fentry, &out.Fentry + *out = new(FentryProgramInfo) + (*in).DeepCopyInto(*out) + } + if in.Fexit != nil { + in, out := &in.Fexit, &out.Fexit + *out = new(FexitProgramInfo) + (*in).DeepCopyInto(*out) + } + if in.Kprobe != nil { + in, out := &in.Kprobe, &out.Kprobe + *out = new(KprobeProgramInfo) + (*in).DeepCopyInto(*out) + } + if in.Kretprobe != nil { + in, out := &in.Kretprobe, &out.Kretprobe + *out = new(KprobeProgramInfo) + (*in).DeepCopyInto(*out) + } + if in.Uprobe != nil { + in, out := &in.Uprobe, &out.Uprobe + *out = new(UprobeProgramInfo) + (*in).DeepCopyInto(*out) + } + if in.Uretprobe != nil { + in, out := &in.Uretprobe, &out.Uretprobe + *out = new(UprobeProgramInfo) + (*in).DeepCopyInto(*out) + } + if in.Tracepoint != nil { + in, out := &in.Tracepoint, &out.Tracepoint + *out = new(TracepointProgramInfo) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BpfApplicationProgram. +func (in *BpfApplicationProgram) DeepCopy() *BpfApplicationProgram { + if in == nil { + return nil + } + out := new(BpfApplicationProgram) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BpfApplicationSpec) DeepCopyInto(out *BpfApplicationSpec) { + *out = *in + in.BpfAppCommon.DeepCopyInto(&out.BpfAppCommon) + if in.Programs != nil { + in, out := &in.Programs, &out.Programs + *out = make([]BpfApplicationProgram, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BpfApplicationSpec. +func (in *BpfApplicationSpec) DeepCopy() *BpfApplicationSpec { + if in == nil { + return nil + } + out := new(BpfApplicationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BpfApplicationStatus) DeepCopyInto(out *BpfApplicationStatus) { + *out = *in + in.BpfProgramStatusCommon.DeepCopyInto(&out.BpfProgramStatusCommon) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BpfApplicationStatus. +func (in *BpfApplicationStatus) DeepCopy() *BpfApplicationStatus { + if in == nil { + return nil + } + out := new(BpfApplicationStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BpfProgram) DeepCopyInto(out *BpfProgram) { *out = *in diff --git a/apis/v1alpha1/zz_generated.register.go b/apis/v1alpha1/zz_generated.register.go index aaee37453..3d3ad9e37 100644 --- a/apis/v1alpha1/zz_generated.register.go +++ b/apis/v1alpha1/zz_generated.register.go @@ -61,6 +61,8 @@ func init() { // Adds the list of known types to Scheme. func addKnownTypes(scheme *runtime.Scheme) error { scheme.AddKnownTypes(SchemeGroupVersion, + &BpfApplication{}, + &BpfApplicationList{}, &BpfProgram{}, &BpfProgramList{}, &FentryProgram{}, diff --git a/bundle/manifests/bpfman-operator.clusterserviceversion.yaml b/bundle/manifests/bpfman-operator.clusterserviceversion.yaml index 726b39d86..80761a387 100644 --- a/bundle/manifests/bpfman-operator.clusterserviceversion.yaml +++ b/bundle/manifests/bpfman-operator.clusterserviceversion.yaml @@ -2,217 +2,7 @@ apiVersion: operators.coreos.com/v1alpha1 kind: ClusterServiceVersion metadata: annotations: - alm-examples: |- - [ - { - "apiVersion": "bpfman.io/v1alpha1", - "kind": "FentryProgram", - "metadata": { - "labels": { - "app.kubernetes.io/name": "fentryprogram" - }, - "name": "fentry-example" - }, - "spec": { - "bpffunctionname": "test_fentry", - "bytecode": { - "image": { - "url": "quay.io/bpfman-bytecode/fentry:latest" - } - }, - "func_name": "do_unlinkat", - "nodeselector": {} - } - }, - { - "apiVersion": "bpfman.io/v1alpha1", - "kind": "FexitProgram", - "metadata": { - "labels": { - "app.kubernetes.io/name": "fexitprogram" - }, - "name": "fexit-example" - }, - "spec": { - "bpffunctionname": "test_fexit", - "bytecode": { - "image": { - "url": "quay.io/bpfman-bytecode/fexit:latest" - } - }, - "func_name": "do_unlinkat", - "nodeselector": {} - } - }, - { - "apiVersion": "bpfman.io/v1alpha1", - "kind": "KprobeProgram", - "metadata": { - "labels": { - "app.kubernetes.io/name": "kprobeprogram" - }, - "name": "kprobe-example" - }, - "spec": { - "bpffunctionname": "my_kprobe", - "bytecode": { - "image": { - "url": "quay.io/bpfman-bytecode/kprobe:latest" - } - }, - "func_name": "try_to_wake_up", - "globaldata": { - "GLOBAL_u32": [ - 13, - 12, - 11, - 10 - ], - "GLOBAL_u8": [ - 1 - ] - }, - "nodeselector": {}, - "offset": 0, - "retprobe": false - } - }, - { - "apiVersion": "bpfman.io/v1alpha1", - "kind": "TcProgram", - "metadata": { - "labels": { - "app.kubernetes.io/name": "tcprogram" - }, - "name": "tc-pass-all-nodes" - }, - "spec": { - "bpffunctionname": "pass", - "bytecode": { - "image": { - "url": "quay.io/bpfman-bytecode/tc_pass:latest" - } - }, - "direction": "ingress", - "globaldata": { - "GLOBAL_u32": [ - 13, - 12, - 11, - 10 - ], - "GLOBAL_u8": [ - 1 - ] - }, - "interfaceselector": { - "primarynodeinterface": true - }, - "nodeselector": {}, - "priority": 0 - } - }, - { - "apiVersion": "bpfman.io/v1alpha1", - "kind": "TracepointProgram", - "metadata": { - "labels": { - "app.kubernetes.io/name": "tracepointprogram" - }, - "name": "tracepoint-example" - }, - "spec": { - "bpffunctionname": "enter_openat", - "bytecode": { - "image": { - "url": "quay.io/bpfman-bytecode/tracepoint:latest" - } - }, - "globaldata": { - "GLOBAL_u32": [ - 13, - 12, - 11, - 10 - ], - "GLOBAL_u8": [ - 1 - ] - }, - "names": [ - "syscalls/sys_enter_openat" - ], - "nodeselector": {} - } - }, - { - "apiVersion": "bpfman.io/v1alpha1", - "kind": "UprobeProgram", - "metadata": { - "labels": { - "app.kubernetes.io/name": "uprobeprogram" - }, - "name": "uprobe-example" - }, - "spec": { - "bpffunctionname": "my_uprobe", - "bytecode": { - "image": { - "url": "quay.io/bpfman-bytecode/uprobe:latest" - } - }, - "func_name": "syscall", - "globaldata": { - "GLOBAL_u32": [ - 13, - 12, - 11, - 10 - ], - "GLOBAL_u8": [ - 1 - ] - }, - "nodeselector": {}, - "retprobe": false, - "target": "libc" - } - }, - { - "apiVersion": "bpfman.io/v1alpha1", - "kind": "XdpProgram", - "metadata": { - "labels": { - "app.kubernetes.io/name": "xdpprogram" - }, - "name": "xdp-pass-all-nodes" - }, - "spec": { - "bpffunctionname": "pass", - "bytecode": { - "image": { - "url": "quay.io/bpfman-bytecode/xdp_pass:latest" - } - }, - "globaldata": { - "GLOBAL_u32": [ - 13, - 12, - 11, - 10 - ], - "GLOBAL_u8": [ - 1 - ] - }, - "interfaceselector": { - "primarynodeinterface": true - }, - "nodeselector": {}, - "priority": 0 - } - } - ] + alm-examples: "[]" capabilities: Basic Install categories: OpenShift Optional containerImage: quay.io/bpfman/bpfman-operator:v0.0.0 @@ -240,49 +30,9 @@ metadata: namespace: placeholder spec: apiservicedefinitions: {} - customresourcedefinitions: - owned: - - description: BpfProgram is the Schema for the BpfProgram API - displayName: Bpf Program - kind: BpfProgram - name: bpfprograms.bpfman.io - version: v1alpha1 - - description: FentryProgram is the Schema for the Fentryprograms API - displayName: Fentry Program - kind: FentryProgram - name: fentryprograms.bpfman.io - version: v1alpha1 - - description: FexitProgram is the Schema for the Fexitprograms API - displayName: Fexit Program - kind: FexitProgram - name: fexitprograms.bpfman.io - version: v1alpha1 - - description: KprobeProgram is the Schema for the Kprobeprograms API - displayName: Kprobe Program - kind: KprobeProgram - name: kprobeprograms.bpfman.io - version: v1alpha1 - - description: TcProgram is the Schema for the Tcprograms API - displayName: Tc Program - kind: TcProgram - name: tcprograms.bpfman.io - version: v1alpha1 - - description: TracepointProgram is the Schema for the Tracepointprograms API - displayName: Tracepoint Program - kind: TracepointProgram - name: tracepointprograms.bpfman.io - version: v1alpha1 - - description: UprobeProgram is the Schema for the Uprobeprograms API - displayName: Uprobe Program - kind: UprobeProgram - name: uprobeprograms.bpfman.io - version: v1alpha1 - - description: XdpProgram is the Schema for the Xdpprograms API - displayName: Xdp Program - kind: XdpProgram - name: xdpprograms.bpfman.io - version: v1alpha1 - description: "The bpfman Operator is a Kubernetes Operator for deploying [bpfman](https://bpfman.netlify.app/), + customresourcedefinitions: {} + description: + "The bpfman Operator is a Kubernetes Operator for deploying [bpfman](https://bpfman.netlify.app/), a system daemon\nfor managing eBPF programs. It deploys bpfman itself along with CRDs to make deploying\neBPF programs in Kubernetes much easier.\n\n## Quick Start\n\nTo get bpfman up and running quickly simply click 'install' to deploy the bpfman-operator @@ -301,723 +51,723 @@ spec: checkout the [bpfman community website](https://bpfman.io/) for more information." displayName: Bpfman Operator icon: - - base64data: | - PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+Cjwh - RE9DVFlQRSBzdmcgUFVCTElDICItLy9XM0MvL0RURCBTVkcgMS4xLy9FTiIgImh0dHA6Ly93d3cu - dzMub3JnL0dyYXBoaWNzL1NWRy8xLjEvRFREL3N2ZzExLmR0ZCI+CjwhLS0gQ3JlYXRlZCB3aXRo - IFZlY3Rvcm5hdG9yIChodHRwOi8vdmVjdG9ybmF0b3IuaW8vKSAtLT4KPHN2ZyBoZWlnaHQ9IjEw - MCUiIHN0cm9rZS1taXRlcmxpbWl0PSIxMCIgc3R5bGU9ImZpbGwtcnVsZTpub256ZXJvO2NsaXAt - cnVsZTpldmVub2RkO3N0cm9rZS1saW5lY2FwOnJvdW5kO3N0cm9rZS1saW5lam9pbjpyb3VuZDsi - IHZlcnNpb249IjEuMSIgdmlld0JveD0iMCAwIDEwMjQgMTAyNCIgd2lkdGg9IjEwMCUiIHhtbDpz - cGFjZT0icHJlc2VydmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgeG1sbnM6 - dmVjdG9ybmF0b3I9Imh0dHA6Ly92ZWN0b3JuYXRvci5pbyIgeG1sbnM6eGxpbms9Imh0dHA6Ly93 - d3cudzMub3JnLzE5OTkveGxpbmsiPgo8ZGVmcz4KPGxpbmVhckdyYWRpZW50IGdyYWRpZW50VHJh - bnNmb3JtPSJtYXRyaXgoMS40NTY4MyAxLjQ1NjgzIC0xLjQ1NjgzIDEuNDU2ODMgNzkxLjI0NCAt - NzA0LjEzMykiIGdyYWRpZW50VW5pdHM9InVzZXJTcGFjZU9uVXNlIiBpZD0iTGluZWFyR3JhZGll - bnQiIHgxPSIzNjguNDYyIiB4Mj0iMjQyLjMzMSIgeTE9IjQyNy4wNTMiIHkyPSI1NTMuNjE0Ij4K - PHN0b3Agb2Zmc2V0PSIwLjQyMjU5MiIgc3RvcC1jb2xvcj0iIzMzMzEyYyIvPgo8c3RvcCBvZmZz - ZXQ9IjEiIHN0b3AtY29sb3I9IiNmM2M2MjIiLz4KPC9saW5lYXJHcmFkaWVudD4KPHBhdGggZD0i - TTQzNS4xMTMgNDQ4LjQxM0M0MzUuMTEzIDM3NS43MjMgNDkzLjc5MyAyNjguNjkyIDUxMS4yMDkg - MjY4LjY5MkM1MjguNjI1IDI2OC42OTIgNTg4Ljg3MyAzNzAuOTU3IDU4OC44NzMgNDQzLjY0NkM1 - ODguODczIDUxNi4zMzYgNTMyLjc4NSA2MDAuNzc3IDUxMS4yMDkgNjAwLjc3N0M0ODkuNjMzIDYw - MC43NzcgNDM1LjExMyA1MjEuMTAzIDQzNS4xMTMgNDQ4LjQxM1oiIGlkPSJGaWxsIi8+CjxwYXRo - IGQ9Ik00MzUuMTEzIDQ0OC40MTNDNDM1LjExMyAzNzUuNzIzIDQ5My43OTMgMjY4LjY5MiA1MTEu - MjA5IDI2OC42OTJDNTI4LjYyNSAyNjguNjkyIDU4OC44NzMgMzcwLjk1NyA1ODguODczIDQ0My42 - NDZDNTg4Ljg3MyA1MTYuMzM2IDUzMi43ODUgNjAwLjc3NyA1MTEuMjA5IDYwMC43NzdDNDg5LjYz - MyA2MDAuNzc3IDQzNS4xMTMgNTIxLjEwMyA0MzUuMTEzIDQ0OC40MTNaIiBpZD0iRmlsbF8yIi8+ - CjxwYXRoIGQ9Ik00MzUuMTEzIDQ0OC40MTNDNDM1LjExMyAzNzUuNzIzIDQ5My43OTMgMjY4LjY5 - MiA1MTEuMjA5IDI2OC42OTJDNTI4LjYyNSAyNjguNjkyIDU4OC44NzMgMzcwLjk1NyA1ODguODcz - IDQ0My42NDZDNTg4Ljg3MyA1MTYuMzM2IDUzMi43ODUgNjAwLjc3NyA1MTEuMjA5IDYwMC43NzdD - NDg5LjYzMyA2MDAuNzc3IDQzNS4xMTMgNTIxLjEwMyA0MzUuMTEzIDQ0OC40MTNaIiBpZD0iRmls - bF8zIi8+CjxwYXRoIGQ9Ik00MzUuMTIgMzY4LjgzOEM0MzUuMTIgMzY4LjgzOCA0NzQuMjE5IDM3 - OC4yNjMgNTEyLjAwNyAzNzguMzM1QzU0OS43OTYgMzc4LjQwNyA1ODguODggMzY5LjM4NSA1ODgu - ODggMzY5LjM4NSIgaWQ9IkZpbGxfNCIvPgo8L2RlZnM+CjxnIGlkPSJMYXllci0xIiB2ZWN0b3Ju - YXRvcjpsYXllck5hbWU9IkxheWVyIDEiPgo8ZyBvcGFjaXR5PSIxIiB2ZWN0b3JuYXRvcjpsYXll - ck5hbWU9Ikdyb3VwIDEyIj4KPHBhdGggZD0iTTE4Ni4yMzMgMzg3LjI3OEMxODYuMjMzIDIwNy4z - NjQgMzMyLjA4NCA2MS41MTM5IDUxMS45OTggNjEuNTEzOUM2OTEuOTE2IDYxLjUxMzkgODM3Ljc2 - NyAyMDcuMzY0IDgzNy43NjcgMzg3LjI3OEM4MzcuNzY3IDU2Ny4xOTMgNjkxLjkxNiA3MTMuMDQz - IDUxMS45OTggNzEzLjA0M0MzMzIuMDg0IDcxMy4wNDMgMTg2LjIzMyA1NjcuMTkzIDE4Ni4yMzMg - Mzg3LjI3OCIgZmlsbD0iI2YyOGYyMiIgZmlsbC1ydWxlPSJub256ZXJvIiBvcGFjaXR5PSIxIiBz - dHJva2U9Im5vbmUiIHZlY3Rvcm5hdG9yOmxheWVyTmFtZT0icGF0aCIvPgo8ZyBvcGFjaXR5PSIx - IiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9Ikdyb3VwIDEzIj4KPHBhdGggZD0iTTIwMS4zMzEgNDg1 - LjQ4NUMyMDQuODQ3IDQ4MS41MzQgMjA4LjE4MiA0NzcuNTkzIDIxMS4yOTQgNDczLjY2QzI3MS4z - ODUgMzk3LjYzNiAyNDkuMTUgMzQzLjc2OSAxOTguMTU5IDI5OS45OTFDMTkwLjQ0MyAzMjcuNzg0 - IDE4Ni4yMzUgMzU3LjAzIDE4Ni4yMzUgMzg3LjI3N0MxODYuMjM1IDQyMS41MDkgMTkxLjU0NCA0 - NTQuNDkgMjAxLjMzMSA0ODUuNDg1IiBmaWxsPSIjZTBmMjIyIiBmaWxsLXJ1bGU9Im5vbnplcm8i - IG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9uZSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJwYXRoIi8+ - CjxwYXRoIGQ9Ik00MjEuMTYzIDQwNS4wOThDNDQzLjY2IDMyMy4yNzkgMzQxLjE0MSAyNTIuNjQx - IDI0Ni4xNzkgMTk5LjA4QzIzOSAyMDkuMTk4IDIzMi4zNDIgMjE5LjcwMyAyMjYuMzM4IDIzMC42 - MjlDMzA5LjI2NSAyNzguMTY5IDM5MC43NjkgMzQyLjM1OSAzNTYuNjU0IDQyMy4zMDFDMzM0LjIy - NSA0NzYuNTA1IDI3OC43MTMgNTEzLjMwNyAyMzIuNzA4IDU1NS4wMDFDMjM4LjYxNCA1NjQuODE4 - IDI0NS4wMjcgNTc0LjI5MiAyNTEuOTA0IDU4My4zOTZDMzA5Ljk0NCA1MjQuODAxIDM5OS4zNTMg - NDg0LjQyNyA0MjEuMTYzIDQwNS4wOTgiIGZpbGw9IiNlMGYyMjIiIGZpbGwtcnVsZT0ibm9uemVy - byIgb3BhY2l0eT0iMSIgc3Ryb2tlPSJub25lIiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9InBhdGgi - Lz4KPHBhdGggZD0iTTU5Ny45MjMgMzIzLjgwN0M2MjkuMjkyIDQ0Mi45OTkgNDU0LjQwMiA1MTku - MzQxIDM4OC45NiA1OTIuMzA1QzM2Ni4zMDQgNjE3LjU2NiAzNDguODAxIDYzOS4xODcgMzM5LjMw - NiA2NjMuNDY0QzM0OC43NTggNjY5LjM4NyAzNTguNTE1IDY3NC44NTggMzY4LjU4NiA2NzkuODAx - QzM3My43NzQgNjU2LjU2NSAzODQuNTUgNjMyLjg1MSA0MDIuODQgNjA4LjIzNUM0NzEuODAyIDUx - NS40MTcgNjUwLjg2NSA0NTUuNTg1IDYyNi4zMTMgMzE4LjkwM0M2MDcuNzE4IDIxNS4zODYgNDk4 - LjE3NiAxNjEuODQyIDQ2MS45MjQgNjUuMzQ5N0M0MzguOTk3IDY4Ljg4NzIgNDE2Ljg5NSA3NC44 - NzQ4IDM5NS44MDggODIuOTI5OEM0NDcuMzAxIDE3My44MTMgNTcwLjgyNiAyMjAuODQxIDU5Ny45 - MjMgMzIzLjgwNyIgZmlsbD0iI2UwZjIyMiIgZmlsbC1ydWxlPSJub256ZXJvIiBvcGFjaXR5PSIx - IiBzdHJva2U9Im5vbmUiIHZlY3Rvcm5hdG9yOmxheWVyTmFtZT0icGF0aCIvPgo8cGF0aCBkPSJN - NzA1LjQ4MSAzMDQuMzA1Qzc0MC4zNzkgNDYwLjIwOSA1MTkuNjMxIDUyMy4yNjUgNDQ2LjAyNiA2 - MzAuMzQ1QzQzMC40ODQgNjUyLjk0OSA0MjMuMDc4IDY3Ni4zNzEgNDIxLjI1OSA3MDAuMTQxQzQz - NC4zMjYgNzAzLjkyMyA0NDcuNjk4IDcwNi45NzUgNDYxLjM4NCA3MDkuMTExQzQ2Mi43NzIgNjg0 - LjQ2OSA0NjkuOCA2NjAuMjI2IDQ4NS4xMTUgNjM2Ljg4N0M1NTguODAxIDUyNC41ODUgNzg2Ljk3 - MiA0NjguMDg4IDc2MC4xMDggMzA4LjUxM0M3NDcuMzg1IDIzMi45MjcgNzAwLjQ0MyAxNzQuMzU0 - IDY3NS4zNDEgMTA1LjQ2NUM2NTAuMDI4IDkwLjc2MDggNjIyLjU5NiA3OS4yOTMgNTkzLjU0OSA3 - MS44MDUzQzYxOC4xODYgMTU1Ljg1OSA2ODUuNzY0IDIxNi4yMzcgNzA1LjQ4MSAzMDQuMzA1IiBm - aWxsPSIjZTBmMjIyIiBmaWxsLXJ1bGU9Im5vbnplcm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9u - ZSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJwYXRoIi8+CjxwYXRoIGQ9Ik0zMjQuMzM5IDYxMS4y - MjNDMzgxLjUzNyA1MzUuNzk2IDU2My4wOCA0NjMuODc3IDU1Mi43MyAzNTIuNzQ4QzU0My41OTIg - MjU0LjY0OCA0MDIuNjYzIDE5NC4wNzYgMzE2LjE1NSAxMjYuOTYyQzMwNi40NDEgMTM0LjI4MiAy - OTcuMiAxNDIuMTc0IDI4OC4zNzUgMTUwLjUxM0MzODUuNjg1IDIxMS44NTQgNTE1Ljk3MSAyNzcu - NDc2IDUxMy40MzUgMzY1LjIzOUM1MTAuMDk5IDQ4MC42MzIgMzQ3LjM3NCA1MzMuNDMyIDI4NC44 - ODEgNjIwLjcxM0MyOTEuNzU0IDYyNy40MDIgMjk4Ljg5MyA2MzMuODEgMzA2LjMzNCA2MzkuODc1 - QzMxMS4xNzQgNjMwLjI4MSAzMTcuMTMxIDYyMC43MjYgMzI0LjMzOSA2MTEuMjIzIiBmaWxsPSIj - ZTBmMjIyIiBmaWxsLXJ1bGU9Im5vbnplcm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9uZSIgdmVj - dG9ybmF0b3I6bGF5ZXJOYW1lPSJwYXRoIi8+CjxwYXRoIGQ9Ik02MzguNDgxIDYyOS41MjRDNjc3 - LjY2OCA1NjEuMTY0IDc1MS4wODggNTI2LjczOSA4MjEuNTI0IDQ4OC44NjZDODIzLjkxIDQ4MS41 - ODkgODI2LjA4NCA0NzQuMjE4IDgyNy45NjMgNDY2LjcyMUM3NTkuNDc4IDUxNC4zNjQgNjc2LjQy - NiA1NDcuODQzIDYyNC42NzkgNjE3LjQzM0M2MDQuOTcxIDY0My45MzYgNTk3Ljc2NyA2NzIuMTkz - IDU5OC41MTEgNzAxLjM0QzYwNi43MTYgNjk5LjA4NCA2MTQuODE0IDY5Ni41NzMgNjIyLjc0NCA2 - OTMuNzA2QzYyMi4yNzYgNjcxLjQ0NiA2MjYuNzkzIDY0OS45MDcgNjM4LjQ4MSA2MjkuNTI0IiBm - aWxsPSIjZTBmMjIyIiBmaWxsLXJ1bGU9Im5vbnplcm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9u - ZSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJwYXRoIi8+CjxwYXRoIGQ9Ik02ODYuNDcxIDY0NC43 - NjRDNjgzLjU3OCA2NTEuNzQ0IDY4MS43IDY1OC44NjIgNjgwLjYxMiA2NjYuMDYyQzczMS43MzYg - NjM1LjA3NiA3NzMuNTgxIDU5MC4zODIgODAxLjIyNyA1MzcuMTI2Qzc0OS41NDkgNTY0LjQzMyA3 - MDYuMTc0IDU5Ny4yMjUgNjg2LjQ3MSA2NDQuNzY0IiBmaWxsPSIjZTBmMjIyIiBmaWxsLXJ1bGU9 - Im5vbnplcm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9uZSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1l - PSJwYXRoIi8+CjxwYXRoIGQ9Ik04MzYuNjY2IDQxMi45MTlDODM3LjMyOCA0MDQuNDQ3IDgzNy43 - NjcgMzk1LjkxNSA4MzcuNzY3IDM4Ny4yOEM4MzcuNzY3IDM3MC42OCA4MzYuNTA3IDM1NC4zODEg - ODM0LjExMyAzMzguNDUxQzgyMi42OCA0NzIuMzY1IDYzNC41MjEgNTIwLjkyMyA1NjMuMzkzIDYy - MC4wNjhDNTQwLjY1OSA2NTEuNzU5IDUzMC41NjIgNjgyLjM3NiA1MjguNjE5IDcxMi42MjNDNTM4 - LjA0MSA3MTIuMTUgNTQ3LjMzNCA3MTEuMTk2IDU1Ni41MzMgNzA5LjkzN0M1NTcuNDc4IDY4Mi44 - MjMgNTY0Ljg1OCA2NTUuNTggNTgxLjg3MSA2MjguMDY3QzYzNy45ODEgNTM3LjMyNSA3NzQuNjg5 - IDQ5OC43MDUgODM2LjY2NiA0MTIuOTE5IiBmaWxsPSIjZTBmMjIyIiBmaWxsLXJ1bGU9Im5vbnpl - cm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9uZSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJwYXRo - Ii8+CjwvZz4KPC9nPgo8ZyBvcGFjaXR5PSIxIiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9Ikdyb3Vw - IDEzIj4KPHBhdGggZD0iTTQzNS4xMTMgNDQ4LjQxM0M0MzUuMTEzIDM3NS43MjMgNDkzLjc5MyAy - NjguNjkyIDUxMS4yMDkgMjY4LjY5MkM1MjguNjI1IDI2OC42OTIgNTg4Ljg3MyAzNzAuOTU3IDU4 - OC44NzMgNDQzLjY0NkM1ODguODczIDUxNi4zMzYgNTMyLjc4NSA2MDAuNzc3IDUxMS4yMDkgNjAw - Ljc3N0M0ODkuNjMzIDYwMC43NzcgNDM1LjExMyA1MjEuMTAzIDQzNS4xMTMgNDQ4LjQxM1oiIGZp - bGw9InVybCgjTGluZWFyR3JhZGllbnQpIiBmaWxsLXJ1bGU9Im5vbnplcm8iIG9wYWNpdHk9IjEi - IHN0cm9rZT0iIzMzMzEyYyIgc3Ryb2tlLWxpbmVjYXA9ImJ1dHQiIHN0cm9rZS1saW5lam9pbj0i - cm91bmQiIHN0cm9rZS13aWR0aD0iMjQuNDgxOSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJPdmFs - IDEiLz4KPHBhdGggZD0iTTQzOS4yMzMgMjY4LjY5MkM0MzkuMjMzIDI2OC42OTIgNDQxLjE1MiAx - OTUuOTMzIDUxMS45OTMgMTk1LjkzM0M1ODIuODM0IDE5NS45MzMgNTg0Ljc1MiAyNjguNjkyIDU4 - NC43NTIgMjY4LjY5Mkw0MzkuMjMzIDI2OC42OTJaIiBmaWxsPSIjZjNjNjIyIiBmaWxsLXJ1bGU9 - Im5vbnplcm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0iIzMzMzEyYyIgc3Ryb2tlLWxpbmVjYXA9ImJ1 - dHQiIHN0cm9rZS1saW5lam9pbj0icm91bmQiIHN0cm9rZS13aWR0aD0iMjQuNDgxOSIgdmVjdG9y - bmF0b3I6bGF5ZXJOYW1lPSJPdmFsIDIiLz4KPHBhdGggZD0iTTQ4MC4xODggMTk1LjkzM0M0ODAu - MTg4IDE5NS45MzMgNDY3LjUxOCAxNjYuMzQ1IDQ0Mi4zMjQgMTc1LjU1OCIgZmlsbD0ibm9uZSIg - b3BhY2l0eT0iMSIgc3Ryb2tlPSIjMzMzMTJjIiBzdHJva2UtbGluZWNhcD0icm91bmQiIHN0cm9r - ZS1saW5lam9pbj0icm91bmQiIHN0cm9rZS13aWR0aD0iMjQuNDgxOSIgdmVjdG9ybmF0b3I6bGF5 - ZXJOYW1lPSJDdXJ2ZSAyIi8+CjxwYXRoIGQ9Ik01NDUuMzM3IDE5NS45MzNDNTQ1LjMzNyAxOTUu - OTMzIDU1OC4wMDYgMTY2LjM0NSA1ODMuMjAxIDE3NS41NTgiIGZpbGw9Im5vbmUiIG9wYWNpdHk9 - IjEiIHN0cm9rZT0iIzMzMzEyYyIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIiBzdHJva2UtbGluZWpv - aW49InJvdW5kIiBzdHJva2Utd2lkdGg9IjI0LjQ4MTkiIHZlY3Rvcm5hdG9yOmxheWVyTmFtZT0i - Q3VydmUgMyIvPgo8ZyBvcGFjaXR5PSIxIiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9Ikdyb3VwIDIi - Pgo8dXNlIGZpbGw9Im5vbmUiIG9wYWNpdHk9IjEiIHN0cm9rZT0iI2YyYWMyMiIgc3Ryb2tlLWxp - bmVjYXA9ImJ1dHQiIHN0cm9rZS1saW5lam9pbj0icm91bmQiIHN0cm9rZS13aWR0aD0iMjQuNDgx - OSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJPdmFsIDUiIHhsaW5rOmhyZWY9IiNGaWxsIi8+Cjxj - bGlwUGF0aCBjbGlwLXJ1bGU9Im5vbnplcm8iIGlkPSJDbGlwUGF0aCI+Cjx1c2UgeGxpbms6aHJl - Zj0iI0ZpbGwiLz4KPC9jbGlwUGF0aD4KPGcgY2xpcC1wYXRoPSJ1cmwoI0NsaXBQYXRoKSI+Cjxw - YXRoIGQ9Ik00MzUuMTEzIDQyNS4yMzhDNDM1LjExMyA0MjUuMjM4IDQ3NC4yMTIgNDM0LjY2MyA1 - MTIgNDM0LjczNUM1NDkuNzg4IDQzNC44MDcgNTg4Ljg3MyA0MjUuNzg1IDU4OC44NzMgNDI1Ljc4 - NSIgZmlsbD0ibm9uZSIgb3BhY2l0eT0iMSIgc3Ryb2tlPSIjZjJkZTIyIiBzdHJva2UtbGluZWNh - cD0iYnV0dCIgc3Ryb2tlLWxpbmVqb2luPSJtaXRlciIgc3Ryb2tlLXdpZHRoPSIyNC40ODE5IiB2 - ZWN0b3JuYXRvcjpsYXllck5hbWU9IkN1cnZlIDQiLz4KPC9nPgo8L2c+CjxnIG9wYWNpdHk9IjEi - IHZlY3Rvcm5hdG9yOmxheWVyTmFtZT0iR3JvdXAgMyI+Cjx1c2UgZmlsbD0ibm9uZSIgb3BhY2l0 - eT0iMSIgc3Ryb2tlPSIjZjNjNjIyIiBzdHJva2UtbGluZWNhcD0iYnV0dCIgc3Ryb2tlLWxpbmVq - b2luPSJyb3VuZCIgc3Ryb2tlLXdpZHRoPSIyNC40ODE5IiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9 - Ik92YWwgNCIgeGxpbms6aHJlZj0iI0ZpbGxfMiIvPgo8Y2xpcFBhdGggY2xpcC1ydWxlPSJub256 - ZXJvIiBpZD0iQ2xpcFBhdGhfMiI+Cjx1c2UgeGxpbms6aHJlZj0iI0ZpbGxfMiIvPgo8L2NsaXBQ - YXRoPgo8ZyBjbGlwLXBhdGg9InVybCgjQ2xpcFBhdGhfMikiPgo8cGF0aCBkPSJNNDM5LjIzMyA0 - ODEuNjUzQzQzOS4yMzMgNDgxLjY1MyA0NzUuNjIgNDkzLjg5MiA1MTIgNDkzLjg5MkM1NDguMzgg - NDkzLjg5MiA1ODQuNzUyIDQ4MS42NTMgNTg0Ljc1MiA0ODEuNjUzIiBmaWxsPSJub25lIiBvcGFj - aXR5PSIxIiBzdHJva2U9IiNmM2M2MjIiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIgc3Ryb2tlLWxp - bmVqb2luPSJtaXRlciIgc3Ryb2tlLXdpZHRoPSIyNC40ODE5IiB2ZWN0b3JuYXRvcjpsYXllck5h - bWU9IkN1cnZlIDUiLz4KPC9nPgo8L2c+CjxnIG9wYWNpdHk9IjEiIHZlY3Rvcm5hdG9yOmxheWVy - TmFtZT0iR3JvdXAgNCI+Cjx1c2UgZmlsbD0ibm9uZSIgb3BhY2l0eT0iMSIgc3Ryb2tlPSIjZjJk - ZTIyIiBzdHJva2UtbGluZWNhcD0iYnV0dCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIgc3Ryb2tl - LXdpZHRoPSIyNC40ODE5IiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9Ik92YWwgMyIgeGxpbms6aHJl - Zj0iI0ZpbGxfMyIvPgo8Y2xpcFBhdGggY2xpcC1ydWxlPSJub256ZXJvIiBpZD0iQ2xpcFBhdGhf - MyI+Cjx1c2UgeGxpbms6aHJlZj0iI0ZpbGxfMyIvPgo8L2NsaXBQYXRoPgo8ZyBjbGlwLXBhdGg9 - InVybCgjQ2xpcFBhdGhfMykiPgo8cGF0aCBkPSJNNDYxLjI1NiA1NDAuMTc0QzQ2MS4yNTYgNTQw - LjE3NCA0ODcuNDU5IDU0OS41ODYgNTExLjk5MyA1NDkuNTIzQzUzNi41MjcgNTQ5LjQ2MSA1NTku - MzkxIDUzOS45MjUgNTU5LjM5MSA1MzkuOTI1IiBmaWxsPSJub25lIiBvcGFjaXR5PSIxIiBzdHJv - a2U9IiNmMmRlMjIiIHN0cm9rZS1saW5lY2FwPSJzcXVhcmUiIHN0cm9rZS1saW5lam9pbj0ibWl0 - ZXIiIHN0cm9rZS13aWR0aD0iMjQuNDgxOSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJDdXJ2ZSA2 - Ii8+CjwvZz4KPC9nPgo8ZyBvcGFjaXR5PSIxIiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9Ikdyb3Vw - IDEiPgo8dXNlIGZpbGw9Im5vbmUiIG9wYWNpdHk9IjEiIHN0cm9rZT0iI2YzYzYyMiIgc3Ryb2tl - LWxpbmVjYXA9ImJ1dHQiIHN0cm9rZS1saW5lam9pbj0ibWl0ZXIiIHN0cm9rZS13aWR0aD0iMjQu - NDgxOSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJDdXJ2ZSA3IiB4bGluazpocmVmPSIjRmlsbF80 - Ii8+CjxjbGlwUGF0aCBjbGlwLXJ1bGU9Im5vbnplcm8iIGlkPSJDbGlwUGF0aF80Ij4KPHVzZSB4 - bGluazpocmVmPSIjRmlsbF80Ii8+CjwvY2xpcFBhdGg+CjxnIGNsaXAtcGF0aD0idXJsKCNDbGlw - UGF0aF80KSI+CjxwYXRoIGQ9Ik00MzUuMTEzIDQ0OC40MTNDNDM1LjExMyAzNzUuNzIzIDQ5My43 - OTMgMjY4LjY5MiA1MTEuMjA5IDI2OC42OTJDNTI4LjYyNSAyNjguNjkyIDU4OC44NzMgMzcwLjk1 - NyA1ODguODczIDQ0My42NDZDNTg4Ljg3MyA1MTYuMzM2IDUzMi43ODUgNjAwLjc3NyA1MTEuMjA5 - IDYwMC43NzdDNDg5LjYzMyA2MDAuNzc3IDQzNS4xMTMgNTIxLjEwMyA0MzUuMTEzIDQ0OC40MTNa - IiBmaWxsPSJub25lIiBvcGFjaXR5PSIxIiBzdHJva2U9IiNmM2M2MjIiIHN0cm9rZS1saW5lY2Fw - PSJidXR0IiBzdHJva2UtbGluZWpvaW49InJvdW5kIiBzdHJva2Utd2lkdGg9IjI0LjQ4MTkiIHZl - Y3Rvcm5hdG9yOmxheWVyTmFtZT0iT3ZhbCA0Ii8+CjwvZz4KPC9nPgo8cGF0aCBkPSJNNDM1LjEx - MyA0NDguNDEzQzQzNS4xMTMgMzc1LjcyMyA0OTMuNzkzIDI2OC42OTIgNTExLjIwOSAyNjguNjky - QzUyOC42MjUgMjY4LjY5MiA1ODguODczIDM3MC45NTcgNTg4Ljg3MyA0NDMuNjQ2QzU4OC44NzMg - NTE2LjMzNiA1MzIuNzg1IDYwMC43NzcgNTExLjIwOSA2MDAuNzc3QzQ4OS42MzMgNjAwLjc3NyA0 - MzUuMTEzIDUyMS4xMDMgNDM1LjExMyA0NDguNDEzWiIgZmlsbD0ibm9uZSIgb3BhY2l0eT0iMSIg - c3Ryb2tlPSIjMzMzMTJjIiBzdHJva2UtbGluZWNhcD0iYnV0dCIgc3Ryb2tlLWxpbmVqb2luPSJy - b3VuZCIgc3Ryb2tlLXdpZHRoPSIyNC40ODE5IiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9Ik92YWwg - MyIvPgo8cGF0aCBkPSJNNDkwLjQ1MyAyNjkuNDQ2QzQ1MC4xMjUgMjY5LjgwMyAzNjEuNTgxIDI4 - NS40MjggMjk4LjI2OSAzMjUuNTU2QzE5NS4wNDcgMzkwLjk3OCAyNjQuMjI0IDQ2OS4wODkgMzI2 - LjIxMSA0NjguMjI5QzQyMi4xODYgNDY2Ljg5NyA1MTEuOTg5IDMwMC4yNjIgNTExLjk4OSAyNzMu - ODU2QzUxMS45ODkgMjcyLjExMiA1MDkuMTIgMjcwLjgzOSA1MDMuOTYxIDI3MC4xMkM1MDAuNDU1 - IDI2OS42MzEgNDk1Ljg5MiAyNjkuMzk4IDQ5MC40NTMgMjY5LjQ0NlpNNTExLjk4OSAyNzMuODU2 - QzUxMS45ODkgMzAwLjI2MiA2MDEuNzkyIDQ2Ni44OTcgNjk3Ljc2NyA0NjguMjI5Qzc1OS43NTQg - NDY5LjA4OSA4MjguOTYzIDM5MC45NzggNzI1Ljc0MSAzMjUuNTU2QzY1NC4yMzkgMjgwLjIzNyA1 - NTAuNTA5IDI2Ni4xOTIgNTIwLjQ0NSAyNzAuMDc2QzUxNS4wMTYgMjcwLjc3NyA1MTEuOTg5IDI3 - Mi4wNjQgNTExLjk4OSAyNzMuODU2WiIgZmlsbD0iI2ZmZmZmZiIgZmlsbC1ydWxlPSJub256ZXJv - IiBvcGFjaXR5PSIxIiBzdHJva2U9IiMzMzMxMmMiIHN0cm9rZS1saW5lY2FwPSJidXR0IiBzdHJv - a2UtbGluZWpvaW49InJvdW5kIiBzdHJva2Utd2lkdGg9IjI0LjQ4MTkiIHZlY3Rvcm5hdG9yOmxh - eWVyTmFtZT0iQ3VydmUgMSIvPgo8L2c+CjxnIG9wYWNpdHk9IjEiIHZlY3Rvcm5hdG9yOmxheWVy - TmFtZT0iR3JvdXAgMTQiPgo8cGF0aCBkPSJNMTU0LjkxOSA5MjQuNTMzQzE0NS4zOTQgOTI0LjUz - MyAxMzcuMTYxIDkyMi41MTcgMTMwLjIyMSA5MTguNDg1QzEyMy4yOCA5MTQuNDU0IDExNy45Njkg - OTA4LjI2NCAxMTQuMjg2IDg5OS45MThDMTEwLjYwMyA4OTEuNTcxIDEwOC43NjEgODgwLjk5MiAx - MDguNzYxIDg2OC4xODJDMTA4Ljc2MSA4NTUuMzQzIDExMC42ODQgODQ0Ljc4NCAxMTQuNTMxIDgz - Ni41MDZDMTE4LjM3NyA4MjguMjI4IDEyMy43ODQgODIyLjA2IDEzMC43NTEgODE4LjAwMkMxMzcu - NzE4IDgxMy45NDQgMTQ1Ljc3NCA4MTEuOTE0IDE1NC45MTkgODExLjkxNEMxNjUuMjY1IDgxMS45 - MTQgMTc0LjU1MSA4MTQuMjIyIDE4Mi43NzggODE4LjgzN0MxOTEuMDA1IDgyMy40NTIgMTk3LjUy - IDgyOS45NjcgMjAyLjMyNCA4MzguMzgxQzIwNy4xMjggODQ2Ljc5NiAyMDkuNTMgODU2LjczIDIw - OS41MyA4NjguMTgyQzIwOS41MyA4NzkuNjM1IDIwNy4xMjggODg5LjU2OSAyMDIuMzI0IDg5Ny45 - ODNDMTk3LjUyIDkwNi4zOTggMTkxLjAwNSA5MTIuOTI3IDE4Mi43NzggOTE3LjU2OUMxNzQuNTUx - IDkyMi4yMTIgMTY1LjI2NSA5MjQuNTMzIDE1NC45MTkgOTI0LjUzM1pNOTAuMzA5NyA5MjIuOTg3 - TDkwLjMwOTcgNzcxLjkxM0wxMjIuMDcyIDc3MS45MTNMMTIyLjA3MiA4MzUuNTcxTDEyMC4wMzYg - ODY4LjA1OEwxMjAuNTcgOTAwLjU0OUwxMjAuNTcgOTIyLjk4N0w5MC4zMDk3IDkyMi45ODdaTTE0 - OS40NTQgODk4LjUyNkMxNTQuNzMgODk4LjUyNiAxNTkuNDYzIDg5Ny4zMjIgMTYzLjY1MiA4OTQu - OTE1QzE2Ny44NDEgODkyLjUwOCAxNzEuMTc5IDg4OS4wMDcgMTczLjY2OCA4ODQuNDEyQzE3Ni4x - NTcgODc5LjgxNyAxNzcuNDAxIDg3NC40MDcgMTc3LjQwMSA4NjguMTgyQzE3Ny40MDEgODYxLjgy - MiAxNzYuMTU3IDg1Ni4zOTEgMTczLjY2OCA4NTEuODkxQzE3MS4xNzkgODQ3LjM5IDE2Ny44NDEg - ODQzLjkzNyAxNjMuNjUyIDg0MS41MjlDMTU5LjQ2MyA4MzkuMTIyIDE1NC43MyA4MzcuOTE5IDE0 - OS40NTQgODM3LjkxOUMxNDQuMTc3IDgzNy45MTkgMTM5LjQ0NCA4MzkuMTIyIDEzNS4yNTQgODQx - LjUyOUMxMzEuMDY0IDg0My45MzcgMTI3LjcyNSA4NDcuMzkgMTI1LjIzNiA4NTEuODkxQzEyMi43 - NDcgODU2LjM5MSAxMjEuNTAzIDg2MS44MjIgMTIxLjUwMyA4NjguMTgyQzEyMS41MDMgODc0LjQw - NyAxMjIuNzQ3IDg3OS44MTcgMTI1LjIzNiA4ODQuNDEyQzEyNy43MjUgODg5LjAwNyAxMzEuMDY0 - IDg5Mi41MDggMTM1LjI1NCA4OTQuOTE1QzEzOS40NDQgODk3LjMyMiAxNDQuMTc3IDg5OC41MjYg - MTQ5LjQ1NCA4OTguNTI2WiIgZmlsbD0iI2YzYzYyMiIgZmlsbC1ydWxlPSJub256ZXJvIiBvcGFj - aXR5PSIxIiBzdHJva2U9Im5vbmUiLz4KPHBhdGggZD0iTTI5NS40NzYgOTI0LjUzM0MyODYuMzMx - IDkyNC41MzMgMjc4LjI3NSA5MjIuNTA0IDI3MS4zMDggOTE4LjQ0NkMyNjQuMzQxIDkxNC4zODcg - MjU4LjkzNCA5MDguMTk4IDI1NS4wODggODk5Ljg3OEMyNTEuMjQyIDg5MS41NTggMjQ5LjMxOSA4 - ODEuMDE5IDI0OS4zMTkgODY4LjI2MkMyNDkuMzE5IDg1NS4zMTYgMjUxLjE2IDg0NC43MDQgMjU0 - Ljg0MyA4MzYuNDI1QzI1OC41MjYgODI4LjE0NiAyNjMuODM4IDgyMS45OTEgMjcwLjc3OCA4MTcu - OTYxQzI3Ny43MTkgODEzLjkzIDI4NS45NTEgODExLjkxNCAyOTUuNDc2IDgxMS45MTRDMzA1Ljgy - MiA4MTEuOTE0IDMxNS4xMDggODE0LjIzNSAzMjMuMzM1IDgxOC44NzdDMzMxLjU2MiA4MjMuNTE4 - IDMzOC4wNzcgODMwLjA0NiAzNDIuODgxIDgzOC40NjFDMzQ3LjY4NSA4NDYuODc2IDM1MC4wODcg - ODU2LjgwOSAzNTAuMDg3IDg2OC4yNjJDMzUwLjA4NyA4NzkuNzE3IDM0Ny42ODUgODg5LjY1MSAz - NDIuODgxIDg5OC4wNjZDMzM4LjA3NyA5MDYuNDgxIDMzMS41NjIgOTEyLjk5NSAzMjMuMzM1IDkx - Ny42MUMzMTUuMTA4IDkyMi4yMjUgMzA1LjgyMiA5MjQuNTMzIDI5NS40NzYgOTI0LjUzM1pNMjMw - Ljg2NyA5NjIuNDg2TDIzMC44NjcgODEzLjQ1N0wyNjEuMTI4IDgxMy40NTdMMjYxLjEyOCA4MzUu - ODk1TDI2MC41OTMgODY4LjM4NkwyNjIuNjI5IDkwMC44NzdMMjYyLjYyOSA5NjIuNDg2TDIzMC44 - NjcgOTYyLjQ4NlpNMjkwLjAxMSA4OTguNTI2QzI5NS4yODggODk4LjUyNiAzMDAuMDIgODk3LjMy - MiAzMDQuMjA5IDg5NC45MTVDMzA4LjM5OCA4OTIuNTA4IDMxMS43MzYgODg5LjAyIDMxNC4yMjUg - ODg0LjQ1MkMzMTYuNzE0IDg3OS44ODMgMzE3Ljk1OSA4NzQuNDg3IDMxNy45NTkgODY4LjI2MkMz - MTcuOTU5IDg2MS45MDEgMzE2LjcxNCA4NTYuNDU4IDMxNC4yMjUgODUxLjkzMUMzMTEuNzM2IDg0 - Ny40MDQgMzA4LjM5OCA4NDMuOTM3IDMwNC4yMDkgODQxLjUyOUMzMDAuMDIgODM5LjEyMiAyOTUu - Mjg4IDgzNy45MTkgMjkwLjAxMSA4MzcuOTE5QzI4NC43MzQgODM3LjkxOSAyODAuMDAxIDgzOS4x - MjIgMjc1LjgxMSA4NDEuNTI5QzI3MS42MjEgODQzLjkzNyAyNjguMjgyIDg0Ny40MDQgMjY1Ljc5 - MyA4NTEuOTMxQzI2My4zMDQgODU2LjQ1OCAyNjIuMDYgODYxLjkwMSAyNjIuMDYgODY4LjI2MkMy - NjIuMDYgODc0LjQ4NyAyNjMuMzA0IDg3OS44ODMgMjY1Ljc5MyA4ODQuNDUyQzI2OC4yODIgODg5 - LjAyIDI3MS42MjEgODkyLjUwOCAyNzUuODExIDg5NC45MTVDMjgwLjAwMSA4OTcuMzIyIDI4NC43 - MzQgODk4LjUyNiAyOTAuMDExIDg5OC41MjZaIiBmaWxsPSIjZjNjNjIyIiBmaWxsLXJ1bGU9Im5v - bnplcm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9uZSIvPgo8cGF0aCBkPSJNMzc1LjI4NiA5MjIu - OTg3TDM3NS4yODYgODEwLjk3QzM3NS4yODYgNzk4LjYzIDM3OC45NDEgNzg4Ljc3OCAzODYuMjQ5 - IDc4MS40MTRDMzkzLjU1OCA3NzQuMDQ5IDQwNC4wMDYgNzcwLjM2NiA0MTcuNTk1IDc3MC4zNjZD - NDIyLjE1MiA3NzAuMzY2IDQyNi41ODEgNzcwLjgyOCA0MzAuODggNzcxLjc1MkM0MzUuMTc5IDc3 - Mi42NzYgNDM4LjgyIDc3NC4xMjkgNDQxLjgwNCA3NzYuMTEyTDQzMy41MTQgNzk5LjExQzQzMS43 - NzUgNzk3LjkzOSA0MjkuNzk4IDc5NyA0MjcuNTgyIDc5Ni4yOTNDNDI1LjM2NyA3OTUuNTg1IDQy - My4wNjMgNzk1LjIzMiA0MjAuNjcxIDc5NS4yMzJDNDE2LjAyMiA3OTUuMjMyIDQxMi40MzMgNzk2 - LjU1NyA0MDkuOTA1IDc5OS4yMDhDNDA3LjM3NyA4MDEuODU5IDQwNi4xMTMgODA1Ljg3NCA0MDYu - MTEzIDgxMS4yNTNMNDA2LjExMyA4MjEuNDYyTDQwNy4wNDkgODM1LjExNkw0MDcuMDQ5IDkyMi45 - ODdMMzc1LjI4NiA5MjIuOTg3Wk0zNTguMjk1IDg0MC4yNkwzNTguMjk1IDgxNS44ODJMNDM0LjI3 - NCA4MTUuODgyTDQzNC4yNzQgODQwLjI2TDM1OC4yOTUgODQwLjI2WiIgZmlsbD0iI2YzYzYyMiIg - ZmlsbC1ydWxlPSJub256ZXJvIiBvcGFjaXR5PSIxIiBzdHJva2U9Im5vbmUiLz4KPC9nPgo8ZyBv - cGFjaXR5PSIxIiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9Ikdyb3VwIDE1Ij4KPHBhdGggZD0iTTYy - Mi45MDYgODExLjkxNEM2MzEuNTY4IDgxMS45MTQgNjM5LjI1OCA4MTMuNjI0IDY0NS45NzcgODE3 - LjA0NEM2NTIuNjk2IDgyMC40NjQgNjU3Ljk2OSA4MjUuNzM3IDY2MS43OTYgODMyLjg2MkM2NjUu - NjIzIDgzOS45ODcgNjY3LjUzNyA4NDkuMTQgNjY3LjUzNyA4NjAuMzIxTDY2Ny41MzcgOTIyLjk4 - N0w2MzUuNzc1IDkyMi45ODdMNjM1Ljc3NSA4NjUuMDY4QzYzNS43NzUgODU2LjI2MiA2MzMuOTQ1 - IDg0OS43NjUgNjMwLjI4NyA4NDUuNTc4QzYyNi42MjggODQxLjM5IDYyMS41MzMgODM5LjI5NiA2 - MTUuMDAxIDgzOS4yOTZDNjEwLjMyIDgzOS4yOTYgNjA2LjE1NyA4NDAuMzQzIDYwMi41MTEgODQy - LjQzOEM1OTguODY1IDg0NC41MzIgNTk2LjAyOSA4NDcuNjczIDU5NC4wMDMgODUxLjg2MUM1OTEu - OTc2IDg1Ni4wNDggNTkwLjk2MyA4NjEuNDMyIDU5MC45NjMgODY4LjAxMUw1OTAuOTYzIDkyMi45 - ODdMNTU5LjE5NyA5MjIuOTg3TDU1OS4xOTcgODY1LjA2OEM1NTkuMTk3IDg1Ni4yNjIgNTU3LjM5 - NSA4NDkuNzY1IDU1My43OTEgODQ1LjU3OEM1NTAuMTg2IDg0MS4zOSA1NDUuMDY0IDgzOS4yOTYg - NTM4LjQyMyA4MzkuMjk2QzUzMy43NDUgODM5LjI5NiA1MjkuNTgyIDg0MC4zNDMgNTI1LjkzNSA4 - NDIuNDM4QzUyMi4yODggODQ0LjUzMiA1MTkuNDUyIDg0Ny42NzMgNTE3LjQyNSA4NTEuODYxQzUx - NS4zOTkgODU2LjA0OCA1MTQuMzg1IDg2MS40MzIgNTE0LjM4NSA4NjguMDExTDUxNC4zODUgOTIy - Ljk4N0w0ODIuNjIzIDkyMi45ODdMNDgyLjYyMyA4MTMuNDU3TDUxMi44ODQgODEzLjQ1N0w1MTIu - ODg0IDg0My4zMzZMNTA3LjExOSA4MzQuNjIzQzUxMC45MTggODI3LjE4OSA1MTYuMzI1IDgyMS41 - NDYgNTIzLjM0MiA4MTcuNjkzQzUzMC4zNTkgODEzLjg0MSA1MzguMzM0IDgxMS45MTQgNTQ3LjI2 - NyA4MTEuOTE0QzU1Ny4yODYgODExLjkxNCA1NjYuMDYzIDgxNC40MzkgNTczLjU5NiA4MTkuNDg4 - QzU4MS4xMjkgODI0LjUzNiA1ODYuMTQ0IDgzMi4zMTIgNTg4LjY0IDg0Mi44MTVMNTc3LjQyIDgz - OS43MTZDNTgxLjA4MSA4MzEuMjI0IDU4Ni45MzYgODI0LjQ2NyA1OTQuOTg3IDgxOS40NDZDNjAz - LjAzOCA4MTQuNDI1IDYxMi4zNDQgODExLjkxNCA2MjIuOTA2IDgxMS45MTRaIiBmaWxsPSIjOTk5 - OTk5IiBmaWxsLXJ1bGU9Im5vbnplcm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9uZSIvPgo8cGF0 - aCBkPSJNNzYzLjIzOCA5MjIuOTg3TDc2My4yMzggOTAxLjY0TDc2MS4zMzMgODk2LjgzNkw3NjEu - MzMzIDg1OC42MzhDNzYxLjMzMyA4NTEuODE2IDc1OS4yNiA4NDYuNTI5IDc1NS4xMTUgODQyLjc3 - OEM3NTAuOTY5IDgzOS4wMjcgNzQ0LjU5MSA4MzcuMTUyIDczNS45ODEgODM3LjE1MkM3MzAuMjIg - ODM3LjE1MiA3MjQuNTE2IDgzOC4wNTUgNzE4Ljg2NyA4MzkuODYxQzcxMy4yMTggODQxLjY2NyA3 - MDguMzk3IDg0NC4xMzIgNzA0LjQwMyA4NDcuMjU2TDY5My4wNjIgODI1LjE1MkM2OTkuMDg1IDgy - MC44MzQgNzA2LjI5NiA4MTcuNTQ4IDcxNC42OTQgODE1LjI5NUM3MjMuMDkzIDgxMy4wNDEgNzMx - LjYzNiA4MTEuOTE0IDc0MC4zMjMgODExLjkxNEM3NTcuMDg3IDgxMS45MTQgNzcwLjA3MyA4MTUu - ODQyIDc3OS4yODIgODIzLjY5NkM3ODguNDkgODMxLjU1MSA3OTMuMDk1IDg0My43OTMgNzkzLjA5 - NSA4NjAuNDIzTDc5My4wOTUgOTIyLjk4N0w3NjMuMjM4IDkyMi45ODdaTTcyOS45NTggOTI0LjUz - M0M3MjEuNDM5IDkyNC41MzMgNzE0LjEyNSA5MjMuMDk1IDcwOC4wMTcgOTIwLjIxOEM3MDEuOTA5 - IDkxNy4zNDEgNjk3LjIzMiA5MTMuMzkyIDY5My45ODcgOTA4LjM3QzY5MC43NDIgOTAzLjM0OSA2 - ODkuMTIgODk3LjY5IDY4OS4xMiA4OTEuMzkzQzY4OS4xMiA4ODQuOTM3IDY5MC43MDYgODc5LjI1 - OSA2OTMuODc5IDg3NC4zNTlDNjk3LjA1MiA4NjkuNDU5IDcwMi4wOTggODY1LjYxNyA3MDkuMDE3 - IDg2Mi44MzNDNzE1LjkzNyA4NjAuMDQ5IDcyNC45NjEgODU4LjY1OCA3MzYuMDg5IDg1OC42NThM - NzY1LjA3NCA4NTguNjU4TDc2NS4wNzQgODc3LjE2Nkw3MzkuNjExIDg3Ny4xNjZDNzMyLjEwMyA4 - NzcuMTY2IDcyNi45NjMgODc4LjM3NyA3MjQuMTkxIDg4MC43OThDNzIxLjQxOSA4ODMuMjE5IDcy - MC4wMzMgODg2LjMxOCA3MjAuMDMzIDg5MC4wOTVDNzIwLjAzMyA4OTQuMDY2IDcyMS42MTEgODk3 - LjI0NyA3MjQuNzY3IDg5OS42NDFDNzI3LjkyMiA5MDIuMDM0IDczMi4yNjIgOTAzLjIzMSA3Mzcu - Nzg0IDkwMy4yMzFDNzQzLjExOCA5MDMuMjMxIDc0Ny45MDcgOTAxLjk5MyA3NTIuMTUxIDg5OS41 - MThDNzU2LjM5NSA4OTcuMDQzIDc1OS40NTYgODkzLjMxOCA3NjEuMzMzIDg4OC4zNDJMNzY2LjEz - IDkwMy4wOTFDNzYzLjg3NSA5MTAuMDYgNzU5LjY5MyA5MTUuMzc2IDc1My41ODEgOTE5LjAzOUM3 - NDcuNDcgOTIyLjcwMiA3MzkuNTk2IDkyNC41MzMgNzI5Ljk1OCA5MjQuNTMzWiIgZmlsbD0iIzk5 - OTk5OSIgZmlsbC1ydWxlPSJub256ZXJvIiBvcGFjaXR5PSIxIiBzdHJva2U9Im5vbmUiLz4KPHBh - dGggZD0iTTg4OC4yMDcgODExLjkxNEM4OTYuOTIyIDgxMS45MTQgOTA0LjY5OSA4MTMuNjI0IDkx - MS41NCA4MTcuMDQ0QzkxOC4zODEgODIwLjQ2NCA5MjMuNzgzIDgyNS43MzcgOTI3Ljc0NiA4MzIu - ODYyQzkzMS43MDkgODM5Ljk4NyA5MzMuNjkgODQ5LjE0IDkzMy42OSA4NjAuMzIxTDkzMy42OSA5 - MjIuOTg3TDkwMS45MjggOTIyLjk4N0w5MDEuOTI4IDg2NS4wNjhDOTAxLjkyOCA4NTYuMjYyIDg5 - OS45OSA4NDkuNzY1IDg5Ni4xMTMgODQ1LjU3OEM4OTIuMjM2IDg0MS4zOSA4ODYuODAxIDgzOS4y - OTYgODc5LjgwOCA4MzkuMjk2Qzg3NC43MiA4MzkuMjk2IDg3MC4xNzcgODQwLjM1NiA4NjYuMTgg - ODQyLjQ3N0M4NjIuMTgyIDg0NC41OTggODU5LjA3NSA4NDcuODIgODU2Ljg1OCA4NTIuMTQyQzg1 - NC42NDIgODU2LjQ2NSA4NTMuNTM0IDg2Mi4wMjMgODUzLjUzNCA4NjguODE5TDg1My41MzQgOTIy - Ljk4N0w4MjEuNzcyIDkyMi45ODdMODIxLjc3MiA4MTMuNDU3TDg1Mi4wMzIgODEzLjQ1N0w4NTIu - MDMyIDg0My44MjNMODQ2LjM0NyA4MzQuNzAyQzg1MC4yODEgODI3LjI3MSA4NTUuOTA3IDgyMS42 - MTUgODYzLjIyMyA4MTcuNzM1Qzg3MC41MzkgODEzLjg1NSA4NzguODY3IDgxMS45MTQgODg4LjIw - NyA4MTEuOTE0WiIgZmlsbD0iIzk5OTk5OSIgZmlsbC1ydWxlPSJub256ZXJvIiBvcGFjaXR5PSIx - IiBzdHJva2U9Im5vbmUiLz4KPC9nPgo8L2c+Cjwvc3ZnPgo= - mediatype: image/svg+xml + - base64data: | + PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+Cjwh + RE9DVFlQRSBzdmcgUFVCTElDICItLy9XM0MvL0RURCBTVkcgMS4xLy9FTiIgImh0dHA6Ly93d3cu + dzMub3JnL0dyYXBoaWNzL1NWRy8xLjEvRFREL3N2ZzExLmR0ZCI+CjwhLS0gQ3JlYXRlZCB3aXRo + IFZlY3Rvcm5hdG9yIChodHRwOi8vdmVjdG9ybmF0b3IuaW8vKSAtLT4KPHN2ZyBoZWlnaHQ9IjEw + MCUiIHN0cm9rZS1taXRlcmxpbWl0PSIxMCIgc3R5bGU9ImZpbGwtcnVsZTpub256ZXJvO2NsaXAt + cnVsZTpldmVub2RkO3N0cm9rZS1saW5lY2FwOnJvdW5kO3N0cm9rZS1saW5lam9pbjpyb3VuZDsi + IHZlcnNpb249IjEuMSIgdmlld0JveD0iMCAwIDEwMjQgMTAyNCIgd2lkdGg9IjEwMCUiIHhtbDpz + cGFjZT0icHJlc2VydmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgeG1sbnM6 + dmVjdG9ybmF0b3I9Imh0dHA6Ly92ZWN0b3JuYXRvci5pbyIgeG1sbnM6eGxpbms9Imh0dHA6Ly93 + d3cudzMub3JnLzE5OTkveGxpbmsiPgo8ZGVmcz4KPGxpbmVhckdyYWRpZW50IGdyYWRpZW50VHJh + bnNmb3JtPSJtYXRyaXgoMS40NTY4MyAxLjQ1NjgzIC0xLjQ1NjgzIDEuNDU2ODMgNzkxLjI0NCAt + NzA0LjEzMykiIGdyYWRpZW50VW5pdHM9InVzZXJTcGFjZU9uVXNlIiBpZD0iTGluZWFyR3JhZGll + bnQiIHgxPSIzNjguNDYyIiB4Mj0iMjQyLjMzMSIgeTE9IjQyNy4wNTMiIHkyPSI1NTMuNjE0Ij4K + PHN0b3Agb2Zmc2V0PSIwLjQyMjU5MiIgc3RvcC1jb2xvcj0iIzMzMzEyYyIvPgo8c3RvcCBvZmZz + ZXQ9IjEiIHN0b3AtY29sb3I9IiNmM2M2MjIiLz4KPC9saW5lYXJHcmFkaWVudD4KPHBhdGggZD0i + TTQzNS4xMTMgNDQ4LjQxM0M0MzUuMTEzIDM3NS43MjMgNDkzLjc5MyAyNjguNjkyIDUxMS4yMDkg + MjY4LjY5MkM1MjguNjI1IDI2OC42OTIgNTg4Ljg3MyAzNzAuOTU3IDU4OC44NzMgNDQzLjY0NkM1 + ODguODczIDUxNi4zMzYgNTMyLjc4NSA2MDAuNzc3IDUxMS4yMDkgNjAwLjc3N0M0ODkuNjMzIDYw + MC43NzcgNDM1LjExMyA1MjEuMTAzIDQzNS4xMTMgNDQ4LjQxM1oiIGlkPSJGaWxsIi8+CjxwYXRo + IGQ9Ik00MzUuMTEzIDQ0OC40MTNDNDM1LjExMyAzNzUuNzIzIDQ5My43OTMgMjY4LjY5MiA1MTEu + MjA5IDI2OC42OTJDNTI4LjYyNSAyNjguNjkyIDU4OC44NzMgMzcwLjk1NyA1ODguODczIDQ0My42 + NDZDNTg4Ljg3MyA1MTYuMzM2IDUzMi43ODUgNjAwLjc3NyA1MTEuMjA5IDYwMC43NzdDNDg5LjYz + MyA2MDAuNzc3IDQzNS4xMTMgNTIxLjEwMyA0MzUuMTEzIDQ0OC40MTNaIiBpZD0iRmlsbF8yIi8+ + CjxwYXRoIGQ9Ik00MzUuMTEzIDQ0OC40MTNDNDM1LjExMyAzNzUuNzIzIDQ5My43OTMgMjY4LjY5 + MiA1MTEuMjA5IDI2OC42OTJDNTI4LjYyNSAyNjguNjkyIDU4OC44NzMgMzcwLjk1NyA1ODguODcz + IDQ0My42NDZDNTg4Ljg3MyA1MTYuMzM2IDUzMi43ODUgNjAwLjc3NyA1MTEuMjA5IDYwMC43NzdD + NDg5LjYzMyA2MDAuNzc3IDQzNS4xMTMgNTIxLjEwMyA0MzUuMTEzIDQ0OC40MTNaIiBpZD0iRmls + bF8zIi8+CjxwYXRoIGQ9Ik00MzUuMTIgMzY4LjgzOEM0MzUuMTIgMzY4LjgzOCA0NzQuMjE5IDM3 + OC4yNjMgNTEyLjAwNyAzNzguMzM1QzU0OS43OTYgMzc4LjQwNyA1ODguODggMzY5LjM4NSA1ODgu + ODggMzY5LjM4NSIgaWQ9IkZpbGxfNCIvPgo8L2RlZnM+CjxnIGlkPSJMYXllci0xIiB2ZWN0b3Ju + YXRvcjpsYXllck5hbWU9IkxheWVyIDEiPgo8ZyBvcGFjaXR5PSIxIiB2ZWN0b3JuYXRvcjpsYXll + ck5hbWU9Ikdyb3VwIDEyIj4KPHBhdGggZD0iTTE4Ni4yMzMgMzg3LjI3OEMxODYuMjMzIDIwNy4z + NjQgMzMyLjA4NCA2MS41MTM5IDUxMS45OTggNjEuNTEzOUM2OTEuOTE2IDYxLjUxMzkgODM3Ljc2 + NyAyMDcuMzY0IDgzNy43NjcgMzg3LjI3OEM4MzcuNzY3IDU2Ny4xOTMgNjkxLjkxNiA3MTMuMDQz + IDUxMS45OTggNzEzLjA0M0MzMzIuMDg0IDcxMy4wNDMgMTg2LjIzMyA1NjcuMTkzIDE4Ni4yMzMg + Mzg3LjI3OCIgZmlsbD0iI2YyOGYyMiIgZmlsbC1ydWxlPSJub256ZXJvIiBvcGFjaXR5PSIxIiBz + dHJva2U9Im5vbmUiIHZlY3Rvcm5hdG9yOmxheWVyTmFtZT0icGF0aCIvPgo8ZyBvcGFjaXR5PSIx + IiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9Ikdyb3VwIDEzIj4KPHBhdGggZD0iTTIwMS4zMzEgNDg1 + LjQ4NUMyMDQuODQ3IDQ4MS41MzQgMjA4LjE4MiA0NzcuNTkzIDIxMS4yOTQgNDczLjY2QzI3MS4z + ODUgMzk3LjYzNiAyNDkuMTUgMzQzLjc2OSAxOTguMTU5IDI5OS45OTFDMTkwLjQ0MyAzMjcuNzg0 + IDE4Ni4yMzUgMzU3LjAzIDE4Ni4yMzUgMzg3LjI3N0MxODYuMjM1IDQyMS41MDkgMTkxLjU0NCA0 + NTQuNDkgMjAxLjMzMSA0ODUuNDg1IiBmaWxsPSIjZTBmMjIyIiBmaWxsLXJ1bGU9Im5vbnplcm8i + IG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9uZSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJwYXRoIi8+ + CjxwYXRoIGQ9Ik00MjEuMTYzIDQwNS4wOThDNDQzLjY2IDMyMy4yNzkgMzQxLjE0MSAyNTIuNjQx + IDI0Ni4xNzkgMTk5LjA4QzIzOSAyMDkuMTk4IDIzMi4zNDIgMjE5LjcwMyAyMjYuMzM4IDIzMC42 + MjlDMzA5LjI2NSAyNzguMTY5IDM5MC43NjkgMzQyLjM1OSAzNTYuNjU0IDQyMy4zMDFDMzM0LjIy + NSA0NzYuNTA1IDI3OC43MTMgNTEzLjMwNyAyMzIuNzA4IDU1NS4wMDFDMjM4LjYxNCA1NjQuODE4 + IDI0NS4wMjcgNTc0LjI5MiAyNTEuOTA0IDU4My4zOTZDMzA5Ljk0NCA1MjQuODAxIDM5OS4zNTMg + NDg0LjQyNyA0MjEuMTYzIDQwNS4wOTgiIGZpbGw9IiNlMGYyMjIiIGZpbGwtcnVsZT0ibm9uemVy + byIgb3BhY2l0eT0iMSIgc3Ryb2tlPSJub25lIiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9InBhdGgi + Lz4KPHBhdGggZD0iTTU5Ny45MjMgMzIzLjgwN0M2MjkuMjkyIDQ0Mi45OTkgNDU0LjQwMiA1MTku + MzQxIDM4OC45NiA1OTIuMzA1QzM2Ni4zMDQgNjE3LjU2NiAzNDguODAxIDYzOS4xODcgMzM5LjMw + NiA2NjMuNDY0QzM0OC43NTggNjY5LjM4NyAzNTguNTE1IDY3NC44NTggMzY4LjU4NiA2NzkuODAx + QzM3My43NzQgNjU2LjU2NSAzODQuNTUgNjMyLjg1MSA0MDIuODQgNjA4LjIzNUM0NzEuODAyIDUx + NS40MTcgNjUwLjg2NSA0NTUuNTg1IDYyNi4zMTMgMzE4LjkwM0M2MDcuNzE4IDIxNS4zODYgNDk4 + LjE3NiAxNjEuODQyIDQ2MS45MjQgNjUuMzQ5N0M0MzguOTk3IDY4Ljg4NzIgNDE2Ljg5NSA3NC44 + NzQ4IDM5NS44MDggODIuOTI5OEM0NDcuMzAxIDE3My44MTMgNTcwLjgyNiAyMjAuODQxIDU5Ny45 + MjMgMzIzLjgwNyIgZmlsbD0iI2UwZjIyMiIgZmlsbC1ydWxlPSJub256ZXJvIiBvcGFjaXR5PSIx + IiBzdHJva2U9Im5vbmUiIHZlY3Rvcm5hdG9yOmxheWVyTmFtZT0icGF0aCIvPgo8cGF0aCBkPSJN + NzA1LjQ4MSAzMDQuMzA1Qzc0MC4zNzkgNDYwLjIwOSA1MTkuNjMxIDUyMy4yNjUgNDQ2LjAyNiA2 + MzAuMzQ1QzQzMC40ODQgNjUyLjk0OSA0MjMuMDc4IDY3Ni4zNzEgNDIxLjI1OSA3MDAuMTQxQzQz + NC4zMjYgNzAzLjkyMyA0NDcuNjk4IDcwNi45NzUgNDYxLjM4NCA3MDkuMTExQzQ2Mi43NzIgNjg0 + LjQ2OSA0NjkuOCA2NjAuMjI2IDQ4NS4xMTUgNjM2Ljg4N0M1NTguODAxIDUyNC41ODUgNzg2Ljk3 + MiA0NjguMDg4IDc2MC4xMDggMzA4LjUxM0M3NDcuMzg1IDIzMi45MjcgNzAwLjQ0MyAxNzQuMzU0 + IDY3NS4zNDEgMTA1LjQ2NUM2NTAuMDI4IDkwLjc2MDggNjIyLjU5NiA3OS4yOTMgNTkzLjU0OSA3 + MS44MDUzQzYxOC4xODYgMTU1Ljg1OSA2ODUuNzY0IDIxNi4yMzcgNzA1LjQ4MSAzMDQuMzA1IiBm + aWxsPSIjZTBmMjIyIiBmaWxsLXJ1bGU9Im5vbnplcm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9u + ZSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJwYXRoIi8+CjxwYXRoIGQ9Ik0zMjQuMzM5IDYxMS4y + MjNDMzgxLjUzNyA1MzUuNzk2IDU2My4wOCA0NjMuODc3IDU1Mi43MyAzNTIuNzQ4QzU0My41OTIg + MjU0LjY0OCA0MDIuNjYzIDE5NC4wNzYgMzE2LjE1NSAxMjYuOTYyQzMwNi40NDEgMTM0LjI4MiAy + OTcuMiAxNDIuMTc0IDI4OC4zNzUgMTUwLjUxM0MzODUuNjg1IDIxMS44NTQgNTE1Ljk3MSAyNzcu + NDc2IDUxMy40MzUgMzY1LjIzOUM1MTAuMDk5IDQ4MC42MzIgMzQ3LjM3NCA1MzMuNDMyIDI4NC44 + ODEgNjIwLjcxM0MyOTEuNzU0IDYyNy40MDIgMjk4Ljg5MyA2MzMuODEgMzA2LjMzNCA2MzkuODc1 + QzMxMS4xNzQgNjMwLjI4MSAzMTcuMTMxIDYyMC43MjYgMzI0LjMzOSA2MTEuMjIzIiBmaWxsPSIj + ZTBmMjIyIiBmaWxsLXJ1bGU9Im5vbnplcm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9uZSIgdmVj + dG9ybmF0b3I6bGF5ZXJOYW1lPSJwYXRoIi8+CjxwYXRoIGQ9Ik02MzguNDgxIDYyOS41MjRDNjc3 + LjY2OCA1NjEuMTY0IDc1MS4wODggNTI2LjczOSA4MjEuNTI0IDQ4OC44NjZDODIzLjkxIDQ4MS41 + ODkgODI2LjA4NCA0NzQuMjE4IDgyNy45NjMgNDY2LjcyMUM3NTkuNDc4IDUxNC4zNjQgNjc2LjQy + NiA1NDcuODQzIDYyNC42NzkgNjE3LjQzM0M2MDQuOTcxIDY0My45MzYgNTk3Ljc2NyA2NzIuMTkz + IDU5OC41MTEgNzAxLjM0QzYwNi43MTYgNjk5LjA4NCA2MTQuODE0IDY5Ni41NzMgNjIyLjc0NCA2 + OTMuNzA2QzYyMi4yNzYgNjcxLjQ0NiA2MjYuNzkzIDY0OS45MDcgNjM4LjQ4MSA2MjkuNTI0IiBm + aWxsPSIjZTBmMjIyIiBmaWxsLXJ1bGU9Im5vbnplcm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9u + ZSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJwYXRoIi8+CjxwYXRoIGQ9Ik02ODYuNDcxIDY0NC43 + NjRDNjgzLjU3OCA2NTEuNzQ0IDY4MS43IDY1OC44NjIgNjgwLjYxMiA2NjYuMDYyQzczMS43MzYg + NjM1LjA3NiA3NzMuNTgxIDU5MC4zODIgODAxLjIyNyA1MzcuMTI2Qzc0OS41NDkgNTY0LjQzMyA3 + MDYuMTc0IDU5Ny4yMjUgNjg2LjQ3MSA2NDQuNzY0IiBmaWxsPSIjZTBmMjIyIiBmaWxsLXJ1bGU9 + Im5vbnplcm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9uZSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1l + PSJwYXRoIi8+CjxwYXRoIGQ9Ik04MzYuNjY2IDQxMi45MTlDODM3LjMyOCA0MDQuNDQ3IDgzNy43 + NjcgMzk1LjkxNSA4MzcuNzY3IDM4Ny4yOEM4MzcuNzY3IDM3MC42OCA4MzYuNTA3IDM1NC4zODEg + ODM0LjExMyAzMzguNDUxQzgyMi42OCA0NzIuMzY1IDYzNC41MjEgNTIwLjkyMyA1NjMuMzkzIDYy + MC4wNjhDNTQwLjY1OSA2NTEuNzU5IDUzMC41NjIgNjgyLjM3NiA1MjguNjE5IDcxMi42MjNDNTM4 + LjA0MSA3MTIuMTUgNTQ3LjMzNCA3MTEuMTk2IDU1Ni41MzMgNzA5LjkzN0M1NTcuNDc4IDY4Mi44 + MjMgNTY0Ljg1OCA2NTUuNTggNTgxLjg3MSA2MjguMDY3QzYzNy45ODEgNTM3LjMyNSA3NzQuNjg5 + IDQ5OC43MDUgODM2LjY2NiA0MTIuOTE5IiBmaWxsPSIjZTBmMjIyIiBmaWxsLXJ1bGU9Im5vbnpl + cm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9uZSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJwYXRo + Ii8+CjwvZz4KPC9nPgo8ZyBvcGFjaXR5PSIxIiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9Ikdyb3Vw + IDEzIj4KPHBhdGggZD0iTTQzNS4xMTMgNDQ4LjQxM0M0MzUuMTEzIDM3NS43MjMgNDkzLjc5MyAy + NjguNjkyIDUxMS4yMDkgMjY4LjY5MkM1MjguNjI1IDI2OC42OTIgNTg4Ljg3MyAzNzAuOTU3IDU4 + OC44NzMgNDQzLjY0NkM1ODguODczIDUxNi4zMzYgNTMyLjc4NSA2MDAuNzc3IDUxMS4yMDkgNjAw + Ljc3N0M0ODkuNjMzIDYwMC43NzcgNDM1LjExMyA1MjEuMTAzIDQzNS4xMTMgNDQ4LjQxM1oiIGZp + bGw9InVybCgjTGluZWFyR3JhZGllbnQpIiBmaWxsLXJ1bGU9Im5vbnplcm8iIG9wYWNpdHk9IjEi + IHN0cm9rZT0iIzMzMzEyYyIgc3Ryb2tlLWxpbmVjYXA9ImJ1dHQiIHN0cm9rZS1saW5lam9pbj0i + cm91bmQiIHN0cm9rZS13aWR0aD0iMjQuNDgxOSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJPdmFs + IDEiLz4KPHBhdGggZD0iTTQzOS4yMzMgMjY4LjY5MkM0MzkuMjMzIDI2OC42OTIgNDQxLjE1MiAx + OTUuOTMzIDUxMS45OTMgMTk1LjkzM0M1ODIuODM0IDE5NS45MzMgNTg0Ljc1MiAyNjguNjkyIDU4 + NC43NTIgMjY4LjY5Mkw0MzkuMjMzIDI2OC42OTJaIiBmaWxsPSIjZjNjNjIyIiBmaWxsLXJ1bGU9 + Im5vbnplcm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0iIzMzMzEyYyIgc3Ryb2tlLWxpbmVjYXA9ImJ1 + dHQiIHN0cm9rZS1saW5lam9pbj0icm91bmQiIHN0cm9rZS13aWR0aD0iMjQuNDgxOSIgdmVjdG9y + bmF0b3I6bGF5ZXJOYW1lPSJPdmFsIDIiLz4KPHBhdGggZD0iTTQ4MC4xODggMTk1LjkzM0M0ODAu + MTg4IDE5NS45MzMgNDY3LjUxOCAxNjYuMzQ1IDQ0Mi4zMjQgMTc1LjU1OCIgZmlsbD0ibm9uZSIg + b3BhY2l0eT0iMSIgc3Ryb2tlPSIjMzMzMTJjIiBzdHJva2UtbGluZWNhcD0icm91bmQiIHN0cm9r + ZS1saW5lam9pbj0icm91bmQiIHN0cm9rZS13aWR0aD0iMjQuNDgxOSIgdmVjdG9ybmF0b3I6bGF5 + ZXJOYW1lPSJDdXJ2ZSAyIi8+CjxwYXRoIGQ9Ik01NDUuMzM3IDE5NS45MzNDNTQ1LjMzNyAxOTUu + OTMzIDU1OC4wMDYgMTY2LjM0NSA1ODMuMjAxIDE3NS41NTgiIGZpbGw9Im5vbmUiIG9wYWNpdHk9 + IjEiIHN0cm9rZT0iIzMzMzEyYyIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIiBzdHJva2UtbGluZWpv + aW49InJvdW5kIiBzdHJva2Utd2lkdGg9IjI0LjQ4MTkiIHZlY3Rvcm5hdG9yOmxheWVyTmFtZT0i + Q3VydmUgMyIvPgo8ZyBvcGFjaXR5PSIxIiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9Ikdyb3VwIDIi + Pgo8dXNlIGZpbGw9Im5vbmUiIG9wYWNpdHk9IjEiIHN0cm9rZT0iI2YyYWMyMiIgc3Ryb2tlLWxp + bmVjYXA9ImJ1dHQiIHN0cm9rZS1saW5lam9pbj0icm91bmQiIHN0cm9rZS13aWR0aD0iMjQuNDgx + OSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJPdmFsIDUiIHhsaW5rOmhyZWY9IiNGaWxsIi8+Cjxj + bGlwUGF0aCBjbGlwLXJ1bGU9Im5vbnplcm8iIGlkPSJDbGlwUGF0aCI+Cjx1c2UgeGxpbms6aHJl + Zj0iI0ZpbGwiLz4KPC9jbGlwUGF0aD4KPGcgY2xpcC1wYXRoPSJ1cmwoI0NsaXBQYXRoKSI+Cjxw + YXRoIGQ9Ik00MzUuMTEzIDQyNS4yMzhDNDM1LjExMyA0MjUuMjM4IDQ3NC4yMTIgNDM0LjY2MyA1 + MTIgNDM0LjczNUM1NDkuNzg4IDQzNC44MDcgNTg4Ljg3MyA0MjUuNzg1IDU4OC44NzMgNDI1Ljc4 + NSIgZmlsbD0ibm9uZSIgb3BhY2l0eT0iMSIgc3Ryb2tlPSIjZjJkZTIyIiBzdHJva2UtbGluZWNh + cD0iYnV0dCIgc3Ryb2tlLWxpbmVqb2luPSJtaXRlciIgc3Ryb2tlLXdpZHRoPSIyNC40ODE5IiB2 + ZWN0b3JuYXRvcjpsYXllck5hbWU9IkN1cnZlIDQiLz4KPC9nPgo8L2c+CjxnIG9wYWNpdHk9IjEi + IHZlY3Rvcm5hdG9yOmxheWVyTmFtZT0iR3JvdXAgMyI+Cjx1c2UgZmlsbD0ibm9uZSIgb3BhY2l0 + eT0iMSIgc3Ryb2tlPSIjZjNjNjIyIiBzdHJva2UtbGluZWNhcD0iYnV0dCIgc3Ryb2tlLWxpbmVq + b2luPSJyb3VuZCIgc3Ryb2tlLXdpZHRoPSIyNC40ODE5IiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9 + Ik92YWwgNCIgeGxpbms6aHJlZj0iI0ZpbGxfMiIvPgo8Y2xpcFBhdGggY2xpcC1ydWxlPSJub256 + ZXJvIiBpZD0iQ2xpcFBhdGhfMiI+Cjx1c2UgeGxpbms6aHJlZj0iI0ZpbGxfMiIvPgo8L2NsaXBQ + YXRoPgo8ZyBjbGlwLXBhdGg9InVybCgjQ2xpcFBhdGhfMikiPgo8cGF0aCBkPSJNNDM5LjIzMyA0 + ODEuNjUzQzQzOS4yMzMgNDgxLjY1MyA0NzUuNjIgNDkzLjg5MiA1MTIgNDkzLjg5MkM1NDguMzgg + NDkzLjg5MiA1ODQuNzUyIDQ4MS42NTMgNTg0Ljc1MiA0ODEuNjUzIiBmaWxsPSJub25lIiBvcGFj + aXR5PSIxIiBzdHJva2U9IiNmM2M2MjIiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIgc3Ryb2tlLWxp + bmVqb2luPSJtaXRlciIgc3Ryb2tlLXdpZHRoPSIyNC40ODE5IiB2ZWN0b3JuYXRvcjpsYXllck5h + bWU9IkN1cnZlIDUiLz4KPC9nPgo8L2c+CjxnIG9wYWNpdHk9IjEiIHZlY3Rvcm5hdG9yOmxheWVy + TmFtZT0iR3JvdXAgNCI+Cjx1c2UgZmlsbD0ibm9uZSIgb3BhY2l0eT0iMSIgc3Ryb2tlPSIjZjJk + ZTIyIiBzdHJva2UtbGluZWNhcD0iYnV0dCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIgc3Ryb2tl + LXdpZHRoPSIyNC40ODE5IiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9Ik92YWwgMyIgeGxpbms6aHJl + Zj0iI0ZpbGxfMyIvPgo8Y2xpcFBhdGggY2xpcC1ydWxlPSJub256ZXJvIiBpZD0iQ2xpcFBhdGhf + MyI+Cjx1c2UgeGxpbms6aHJlZj0iI0ZpbGxfMyIvPgo8L2NsaXBQYXRoPgo8ZyBjbGlwLXBhdGg9 + InVybCgjQ2xpcFBhdGhfMykiPgo8cGF0aCBkPSJNNDYxLjI1NiA1NDAuMTc0QzQ2MS4yNTYgNTQw + LjE3NCA0ODcuNDU5IDU0OS41ODYgNTExLjk5MyA1NDkuNTIzQzUzNi41MjcgNTQ5LjQ2MSA1NTku + MzkxIDUzOS45MjUgNTU5LjM5MSA1MzkuOTI1IiBmaWxsPSJub25lIiBvcGFjaXR5PSIxIiBzdHJv + a2U9IiNmMmRlMjIiIHN0cm9rZS1saW5lY2FwPSJzcXVhcmUiIHN0cm9rZS1saW5lam9pbj0ibWl0 + ZXIiIHN0cm9rZS13aWR0aD0iMjQuNDgxOSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJDdXJ2ZSA2 + Ii8+CjwvZz4KPC9nPgo8ZyBvcGFjaXR5PSIxIiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9Ikdyb3Vw + IDEiPgo8dXNlIGZpbGw9Im5vbmUiIG9wYWNpdHk9IjEiIHN0cm9rZT0iI2YzYzYyMiIgc3Ryb2tl + LWxpbmVjYXA9ImJ1dHQiIHN0cm9rZS1saW5lam9pbj0ibWl0ZXIiIHN0cm9rZS13aWR0aD0iMjQu + NDgxOSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJDdXJ2ZSA3IiB4bGluazpocmVmPSIjRmlsbF80 + Ii8+CjxjbGlwUGF0aCBjbGlwLXJ1bGU9Im5vbnplcm8iIGlkPSJDbGlwUGF0aF80Ij4KPHVzZSB4 + bGluazpocmVmPSIjRmlsbF80Ii8+CjwvY2xpcFBhdGg+CjxnIGNsaXAtcGF0aD0idXJsKCNDbGlw + UGF0aF80KSI+CjxwYXRoIGQ9Ik00MzUuMTEzIDQ0OC40MTNDNDM1LjExMyAzNzUuNzIzIDQ5My43 + OTMgMjY4LjY5MiA1MTEuMjA5IDI2OC42OTJDNTI4LjYyNSAyNjguNjkyIDU4OC44NzMgMzcwLjk1 + NyA1ODguODczIDQ0My42NDZDNTg4Ljg3MyA1MTYuMzM2IDUzMi43ODUgNjAwLjc3NyA1MTEuMjA5 + IDYwMC43NzdDNDg5LjYzMyA2MDAuNzc3IDQzNS4xMTMgNTIxLjEwMyA0MzUuMTEzIDQ0OC40MTNa + IiBmaWxsPSJub25lIiBvcGFjaXR5PSIxIiBzdHJva2U9IiNmM2M2MjIiIHN0cm9rZS1saW5lY2Fw + PSJidXR0IiBzdHJva2UtbGluZWpvaW49InJvdW5kIiBzdHJva2Utd2lkdGg9IjI0LjQ4MTkiIHZl + Y3Rvcm5hdG9yOmxheWVyTmFtZT0iT3ZhbCA0Ii8+CjwvZz4KPC9nPgo8cGF0aCBkPSJNNDM1LjEx + MyA0NDguNDEzQzQzNS4xMTMgMzc1LjcyMyA0OTMuNzkzIDI2OC42OTIgNTExLjIwOSAyNjguNjky + QzUyOC42MjUgMjY4LjY5MiA1ODguODczIDM3MC45NTcgNTg4Ljg3MyA0NDMuNjQ2QzU4OC44NzMg + NTE2LjMzNiA1MzIuNzg1IDYwMC43NzcgNTExLjIwOSA2MDAuNzc3QzQ4OS42MzMgNjAwLjc3NyA0 + MzUuMTEzIDUyMS4xMDMgNDM1LjExMyA0NDguNDEzWiIgZmlsbD0ibm9uZSIgb3BhY2l0eT0iMSIg + c3Ryb2tlPSIjMzMzMTJjIiBzdHJva2UtbGluZWNhcD0iYnV0dCIgc3Ryb2tlLWxpbmVqb2luPSJy + b3VuZCIgc3Ryb2tlLXdpZHRoPSIyNC40ODE5IiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9Ik92YWwg + MyIvPgo8cGF0aCBkPSJNNDkwLjQ1MyAyNjkuNDQ2QzQ1MC4xMjUgMjY5LjgwMyAzNjEuNTgxIDI4 + NS40MjggMjk4LjI2OSAzMjUuNTU2QzE5NS4wNDcgMzkwLjk3OCAyNjQuMjI0IDQ2OS4wODkgMzI2 + LjIxMSA0NjguMjI5QzQyMi4xODYgNDY2Ljg5NyA1MTEuOTg5IDMwMC4yNjIgNTExLjk4OSAyNzMu + ODU2QzUxMS45ODkgMjcyLjExMiA1MDkuMTIgMjcwLjgzOSA1MDMuOTYxIDI3MC4xMkM1MDAuNDU1 + IDI2OS42MzEgNDk1Ljg5MiAyNjkuMzk4IDQ5MC40NTMgMjY5LjQ0NlpNNTExLjk4OSAyNzMuODU2 + QzUxMS45ODkgMzAwLjI2MiA2MDEuNzkyIDQ2Ni44OTcgNjk3Ljc2NyA0NjguMjI5Qzc1OS43NTQg + NDY5LjA4OSA4MjguOTYzIDM5MC45NzggNzI1Ljc0MSAzMjUuNTU2QzY1NC4yMzkgMjgwLjIzNyA1 + NTAuNTA5IDI2Ni4xOTIgNTIwLjQ0NSAyNzAuMDc2QzUxNS4wMTYgMjcwLjc3NyA1MTEuOTg5IDI3 + Mi4wNjQgNTExLjk4OSAyNzMuODU2WiIgZmlsbD0iI2ZmZmZmZiIgZmlsbC1ydWxlPSJub256ZXJv + IiBvcGFjaXR5PSIxIiBzdHJva2U9IiMzMzMxMmMiIHN0cm9rZS1saW5lY2FwPSJidXR0IiBzdHJv + a2UtbGluZWpvaW49InJvdW5kIiBzdHJva2Utd2lkdGg9IjI0LjQ4MTkiIHZlY3Rvcm5hdG9yOmxh + eWVyTmFtZT0iQ3VydmUgMSIvPgo8L2c+CjxnIG9wYWNpdHk9IjEiIHZlY3Rvcm5hdG9yOmxheWVy + TmFtZT0iR3JvdXAgMTQiPgo8cGF0aCBkPSJNMTU0LjkxOSA5MjQuNTMzQzE0NS4zOTQgOTI0LjUz + MyAxMzcuMTYxIDkyMi41MTcgMTMwLjIyMSA5MTguNDg1QzEyMy4yOCA5MTQuNDU0IDExNy45Njkg + OTA4LjI2NCAxMTQuMjg2IDg5OS45MThDMTEwLjYwMyA4OTEuNTcxIDEwOC43NjEgODgwLjk5MiAx + MDguNzYxIDg2OC4xODJDMTA4Ljc2MSA4NTUuMzQzIDExMC42ODQgODQ0Ljc4NCAxMTQuNTMxIDgz + Ni41MDZDMTE4LjM3NyA4MjguMjI4IDEyMy43ODQgODIyLjA2IDEzMC43NTEgODE4LjAwMkMxMzcu + NzE4IDgxMy45NDQgMTQ1Ljc3NCA4MTEuOTE0IDE1NC45MTkgODExLjkxNEMxNjUuMjY1IDgxMS45 + MTQgMTc0LjU1MSA4MTQuMjIyIDE4Mi43NzggODE4LjgzN0MxOTEuMDA1IDgyMy40NTIgMTk3LjUy + IDgyOS45NjcgMjAyLjMyNCA4MzguMzgxQzIwNy4xMjggODQ2Ljc5NiAyMDkuNTMgODU2LjczIDIw + OS41MyA4NjguMTgyQzIwOS41MyA4NzkuNjM1IDIwNy4xMjggODg5LjU2OSAyMDIuMzI0IDg5Ny45 + ODNDMTk3LjUyIDkwNi4zOTggMTkxLjAwNSA5MTIuOTI3IDE4Mi43NzggOTE3LjU2OUMxNzQuNTUx + IDkyMi4yMTIgMTY1LjI2NSA5MjQuNTMzIDE1NC45MTkgOTI0LjUzM1pNOTAuMzA5NyA5MjIuOTg3 + TDkwLjMwOTcgNzcxLjkxM0wxMjIuMDcyIDc3MS45MTNMMTIyLjA3MiA4MzUuNTcxTDEyMC4wMzYg + ODY4LjA1OEwxMjAuNTcgOTAwLjU0OUwxMjAuNTcgOTIyLjk4N0w5MC4zMDk3IDkyMi45ODdaTTE0 + OS40NTQgODk4LjUyNkMxNTQuNzMgODk4LjUyNiAxNTkuNDYzIDg5Ny4zMjIgMTYzLjY1MiA4OTQu + OTE1QzE2Ny44NDEgODkyLjUwOCAxNzEuMTc5IDg4OS4wMDcgMTczLjY2OCA4ODQuNDEyQzE3Ni4x + NTcgODc5LjgxNyAxNzcuNDAxIDg3NC40MDcgMTc3LjQwMSA4NjguMTgyQzE3Ny40MDEgODYxLjgy + MiAxNzYuMTU3IDg1Ni4zOTEgMTczLjY2OCA4NTEuODkxQzE3MS4xNzkgODQ3LjM5IDE2Ny44NDEg + ODQzLjkzNyAxNjMuNjUyIDg0MS41MjlDMTU5LjQ2MyA4MzkuMTIyIDE1NC43MyA4MzcuOTE5IDE0 + OS40NTQgODM3LjkxOUMxNDQuMTc3IDgzNy45MTkgMTM5LjQ0NCA4MzkuMTIyIDEzNS4yNTQgODQx + LjUyOUMxMzEuMDY0IDg0My45MzcgMTI3LjcyNSA4NDcuMzkgMTI1LjIzNiA4NTEuODkxQzEyMi43 + NDcgODU2LjM5MSAxMjEuNTAzIDg2MS44MjIgMTIxLjUwMyA4NjguMTgyQzEyMS41MDMgODc0LjQw + NyAxMjIuNzQ3IDg3OS44MTcgMTI1LjIzNiA4ODQuNDEyQzEyNy43MjUgODg5LjAwNyAxMzEuMDY0 + IDg5Mi41MDggMTM1LjI1NCA4OTQuOTE1QzEzOS40NDQgODk3LjMyMiAxNDQuMTc3IDg5OC41MjYg + MTQ5LjQ1NCA4OTguNTI2WiIgZmlsbD0iI2YzYzYyMiIgZmlsbC1ydWxlPSJub256ZXJvIiBvcGFj + aXR5PSIxIiBzdHJva2U9Im5vbmUiLz4KPHBhdGggZD0iTTI5NS40NzYgOTI0LjUzM0MyODYuMzMx + IDkyNC41MzMgMjc4LjI3NSA5MjIuNTA0IDI3MS4zMDggOTE4LjQ0NkMyNjQuMzQxIDkxNC4zODcg + MjU4LjkzNCA5MDguMTk4IDI1NS4wODggODk5Ljg3OEMyNTEuMjQyIDg5MS41NTggMjQ5LjMxOSA4 + ODEuMDE5IDI0OS4zMTkgODY4LjI2MkMyNDkuMzE5IDg1NS4zMTYgMjUxLjE2IDg0NC43MDQgMjU0 + Ljg0MyA4MzYuNDI1QzI1OC41MjYgODI4LjE0NiAyNjMuODM4IDgyMS45OTEgMjcwLjc3OCA4MTcu + OTYxQzI3Ny43MTkgODEzLjkzIDI4NS45NTEgODExLjkxNCAyOTUuNDc2IDgxMS45MTRDMzA1Ljgy + MiA4MTEuOTE0IDMxNS4xMDggODE0LjIzNSAzMjMuMzM1IDgxOC44NzdDMzMxLjU2MiA4MjMuNTE4 + IDMzOC4wNzcgODMwLjA0NiAzNDIuODgxIDgzOC40NjFDMzQ3LjY4NSA4NDYuODc2IDM1MC4wODcg + ODU2LjgwOSAzNTAuMDg3IDg2OC4yNjJDMzUwLjA4NyA4NzkuNzE3IDM0Ny42ODUgODg5LjY1MSAz + NDIuODgxIDg5OC4wNjZDMzM4LjA3NyA5MDYuNDgxIDMzMS41NjIgOTEyLjk5NSAzMjMuMzM1IDkx + Ny42MUMzMTUuMTA4IDkyMi4yMjUgMzA1LjgyMiA5MjQuNTMzIDI5NS40NzYgOTI0LjUzM1pNMjMw + Ljg2NyA5NjIuNDg2TDIzMC44NjcgODEzLjQ1N0wyNjEuMTI4IDgxMy40NTdMMjYxLjEyOCA4MzUu + ODk1TDI2MC41OTMgODY4LjM4NkwyNjIuNjI5IDkwMC44NzdMMjYyLjYyOSA5NjIuNDg2TDIzMC44 + NjcgOTYyLjQ4NlpNMjkwLjAxMSA4OTguNTI2QzI5NS4yODggODk4LjUyNiAzMDAuMDIgODk3LjMy + MiAzMDQuMjA5IDg5NC45MTVDMzA4LjM5OCA4OTIuNTA4IDMxMS43MzYgODg5LjAyIDMxNC4yMjUg + ODg0LjQ1MkMzMTYuNzE0IDg3OS44ODMgMzE3Ljk1OSA4NzQuNDg3IDMxNy45NTkgODY4LjI2MkMz + MTcuOTU5IDg2MS45MDEgMzE2LjcxNCA4NTYuNDU4IDMxNC4yMjUgODUxLjkzMUMzMTEuNzM2IDg0 + Ny40MDQgMzA4LjM5OCA4NDMuOTM3IDMwNC4yMDkgODQxLjUyOUMzMDAuMDIgODM5LjEyMiAyOTUu + Mjg4IDgzNy45MTkgMjkwLjAxMSA4MzcuOTE5QzI4NC43MzQgODM3LjkxOSAyODAuMDAxIDgzOS4x + MjIgMjc1LjgxMSA4NDEuNTI5QzI3MS42MjEgODQzLjkzNyAyNjguMjgyIDg0Ny40MDQgMjY1Ljc5 + MyA4NTEuOTMxQzI2My4zMDQgODU2LjQ1OCAyNjIuMDYgODYxLjkwMSAyNjIuMDYgODY4LjI2MkMy + NjIuMDYgODc0LjQ4NyAyNjMuMzA0IDg3OS44ODMgMjY1Ljc5MyA4ODQuNDUyQzI2OC4yODIgODg5 + LjAyIDI3MS42MjEgODkyLjUwOCAyNzUuODExIDg5NC45MTVDMjgwLjAwMSA4OTcuMzIyIDI4NC43 + MzQgODk4LjUyNiAyOTAuMDExIDg5OC41MjZaIiBmaWxsPSIjZjNjNjIyIiBmaWxsLXJ1bGU9Im5v + bnplcm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9uZSIvPgo8cGF0aCBkPSJNMzc1LjI4NiA5MjIu + OTg3TDM3NS4yODYgODEwLjk3QzM3NS4yODYgNzk4LjYzIDM3OC45NDEgNzg4Ljc3OCAzODYuMjQ5 + IDc4MS40MTRDMzkzLjU1OCA3NzQuMDQ5IDQwNC4wMDYgNzcwLjM2NiA0MTcuNTk1IDc3MC4zNjZD + NDIyLjE1MiA3NzAuMzY2IDQyNi41ODEgNzcwLjgyOCA0MzAuODggNzcxLjc1MkM0MzUuMTc5IDc3 + Mi42NzYgNDM4LjgyIDc3NC4xMjkgNDQxLjgwNCA3NzYuMTEyTDQzMy41MTQgNzk5LjExQzQzMS43 + NzUgNzk3LjkzOSA0MjkuNzk4IDc5NyA0MjcuNTgyIDc5Ni4yOTNDNDI1LjM2NyA3OTUuNTg1IDQy + My4wNjMgNzk1LjIzMiA0MjAuNjcxIDc5NS4yMzJDNDE2LjAyMiA3OTUuMjMyIDQxMi40MzMgNzk2 + LjU1NyA0MDkuOTA1IDc5OS4yMDhDNDA3LjM3NyA4MDEuODU5IDQwNi4xMTMgODA1Ljg3NCA0MDYu + MTEzIDgxMS4yNTNMNDA2LjExMyA4MjEuNDYyTDQwNy4wNDkgODM1LjExNkw0MDcuMDQ5IDkyMi45 + ODdMMzc1LjI4NiA5MjIuOTg3Wk0zNTguMjk1IDg0MC4yNkwzNTguMjk1IDgxNS44ODJMNDM0LjI3 + NCA4MTUuODgyTDQzNC4yNzQgODQwLjI2TDM1OC4yOTUgODQwLjI2WiIgZmlsbD0iI2YzYzYyMiIg + ZmlsbC1ydWxlPSJub256ZXJvIiBvcGFjaXR5PSIxIiBzdHJva2U9Im5vbmUiLz4KPC9nPgo8ZyBv + cGFjaXR5PSIxIiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9Ikdyb3VwIDE1Ij4KPHBhdGggZD0iTTYy + Mi45MDYgODExLjkxNEM2MzEuNTY4IDgxMS45MTQgNjM5LjI1OCA4MTMuNjI0IDY0NS45NzcgODE3 + LjA0NEM2NTIuNjk2IDgyMC40NjQgNjU3Ljk2OSA4MjUuNzM3IDY2MS43OTYgODMyLjg2MkM2NjUu + NjIzIDgzOS45ODcgNjY3LjUzNyA4NDkuMTQgNjY3LjUzNyA4NjAuMzIxTDY2Ny41MzcgOTIyLjk4 + N0w2MzUuNzc1IDkyMi45ODdMNjM1Ljc3NSA4NjUuMDY4QzYzNS43NzUgODU2LjI2MiA2MzMuOTQ1 + IDg0OS43NjUgNjMwLjI4NyA4NDUuNTc4QzYyNi42MjggODQxLjM5IDYyMS41MzMgODM5LjI5NiA2 + MTUuMDAxIDgzOS4yOTZDNjEwLjMyIDgzOS4yOTYgNjA2LjE1NyA4NDAuMzQzIDYwMi41MTEgODQy + LjQzOEM1OTguODY1IDg0NC41MzIgNTk2LjAyOSA4NDcuNjczIDU5NC4wMDMgODUxLjg2MUM1OTEu + OTc2IDg1Ni4wNDggNTkwLjk2MyA4NjEuNDMyIDU5MC45NjMgODY4LjAxMUw1OTAuOTYzIDkyMi45 + ODdMNTU5LjE5NyA5MjIuOTg3TDU1OS4xOTcgODY1LjA2OEM1NTkuMTk3IDg1Ni4yNjIgNTU3LjM5 + NSA4NDkuNzY1IDU1My43OTEgODQ1LjU3OEM1NTAuMTg2IDg0MS4zOSA1NDUuMDY0IDgzOS4yOTYg + NTM4LjQyMyA4MzkuMjk2QzUzMy43NDUgODM5LjI5NiA1MjkuNTgyIDg0MC4zNDMgNTI1LjkzNSA4 + NDIuNDM4QzUyMi4yODggODQ0LjUzMiA1MTkuNDUyIDg0Ny42NzMgNTE3LjQyNSA4NTEuODYxQzUx + NS4zOTkgODU2LjA0OCA1MTQuMzg1IDg2MS40MzIgNTE0LjM4NSA4NjguMDExTDUxNC4zODUgOTIy + Ljk4N0w0ODIuNjIzIDkyMi45ODdMNDgyLjYyMyA4MTMuNDU3TDUxMi44ODQgODEzLjQ1N0w1MTIu + ODg0IDg0My4zMzZMNTA3LjExOSA4MzQuNjIzQzUxMC45MTggODI3LjE4OSA1MTYuMzI1IDgyMS41 + NDYgNTIzLjM0MiA4MTcuNjkzQzUzMC4zNTkgODEzLjg0MSA1MzguMzM0IDgxMS45MTQgNTQ3LjI2 + NyA4MTEuOTE0QzU1Ny4yODYgODExLjkxNCA1NjYuMDYzIDgxNC40MzkgNTczLjU5NiA4MTkuNDg4 + QzU4MS4xMjkgODI0LjUzNiA1ODYuMTQ0IDgzMi4zMTIgNTg4LjY0IDg0Mi44MTVMNTc3LjQyIDgz + OS43MTZDNTgxLjA4MSA4MzEuMjI0IDU4Ni45MzYgODI0LjQ2NyA1OTQuOTg3IDgxOS40NDZDNjAz + LjAzOCA4MTQuNDI1IDYxMi4zNDQgODExLjkxNCA2MjIuOTA2IDgxMS45MTRaIiBmaWxsPSIjOTk5 + OTk5IiBmaWxsLXJ1bGU9Im5vbnplcm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9uZSIvPgo8cGF0 + aCBkPSJNNzYzLjIzOCA5MjIuOTg3TDc2My4yMzggOTAxLjY0TDc2MS4zMzMgODk2LjgzNkw3NjEu + MzMzIDg1OC42MzhDNzYxLjMzMyA4NTEuODE2IDc1OS4yNiA4NDYuNTI5IDc1NS4xMTUgODQyLjc3 + OEM3NTAuOTY5IDgzOS4wMjcgNzQ0LjU5MSA4MzcuMTUyIDczNS45ODEgODM3LjE1MkM3MzAuMjIg + ODM3LjE1MiA3MjQuNTE2IDgzOC4wNTUgNzE4Ljg2NyA4MzkuODYxQzcxMy4yMTggODQxLjY2NyA3 + MDguMzk3IDg0NC4xMzIgNzA0LjQwMyA4NDcuMjU2TDY5My4wNjIgODI1LjE1MkM2OTkuMDg1IDgy + MC44MzQgNzA2LjI5NiA4MTcuNTQ4IDcxNC42OTQgODE1LjI5NUM3MjMuMDkzIDgxMy4wNDEgNzMx + LjYzNiA4MTEuOTE0IDc0MC4zMjMgODExLjkxNEM3NTcuMDg3IDgxMS45MTQgNzcwLjA3MyA4MTUu + ODQyIDc3OS4yODIgODIzLjY5NkM3ODguNDkgODMxLjU1MSA3OTMuMDk1IDg0My43OTMgNzkzLjA5 + NSA4NjAuNDIzTDc5My4wOTUgOTIyLjk4N0w3NjMuMjM4IDkyMi45ODdaTTcyOS45NTggOTI0LjUz + M0M3MjEuNDM5IDkyNC41MzMgNzE0LjEyNSA5MjMuMDk1IDcwOC4wMTcgOTIwLjIxOEM3MDEuOTA5 + IDkxNy4zNDEgNjk3LjIzMiA5MTMuMzkyIDY5My45ODcgOTA4LjM3QzY5MC43NDIgOTAzLjM0OSA2 + ODkuMTIgODk3LjY5IDY4OS4xMiA4OTEuMzkzQzY4OS4xMiA4ODQuOTM3IDY5MC43MDYgODc5LjI1 + OSA2OTMuODc5IDg3NC4zNTlDNjk3LjA1MiA4NjkuNDU5IDcwMi4wOTggODY1LjYxNyA3MDkuMDE3 + IDg2Mi44MzNDNzE1LjkzNyA4NjAuMDQ5IDcyNC45NjEgODU4LjY1OCA3MzYuMDg5IDg1OC42NThM + NzY1LjA3NCA4NTguNjU4TDc2NS4wNzQgODc3LjE2Nkw3MzkuNjExIDg3Ny4xNjZDNzMyLjEwMyA4 + NzcuMTY2IDcyNi45NjMgODc4LjM3NyA3MjQuMTkxIDg4MC43OThDNzIxLjQxOSA4ODMuMjE5IDcy + MC4wMzMgODg2LjMxOCA3MjAuMDMzIDg5MC4wOTVDNzIwLjAzMyA4OTQuMDY2IDcyMS42MTEgODk3 + LjI0NyA3MjQuNzY3IDg5OS42NDFDNzI3LjkyMiA5MDIuMDM0IDczMi4yNjIgOTAzLjIzMSA3Mzcu + Nzg0IDkwMy4yMzFDNzQzLjExOCA5MDMuMjMxIDc0Ny45MDcgOTAxLjk5MyA3NTIuMTUxIDg5OS41 + MThDNzU2LjM5NSA4OTcuMDQzIDc1OS40NTYgODkzLjMxOCA3NjEuMzMzIDg4OC4zNDJMNzY2LjEz + IDkwMy4wOTFDNzYzLjg3NSA5MTAuMDYgNzU5LjY5MyA5MTUuMzc2IDc1My41ODEgOTE5LjAzOUM3 + NDcuNDcgOTIyLjcwMiA3MzkuNTk2IDkyNC41MzMgNzI5Ljk1OCA5MjQuNTMzWiIgZmlsbD0iIzk5 + OTk5OSIgZmlsbC1ydWxlPSJub256ZXJvIiBvcGFjaXR5PSIxIiBzdHJva2U9Im5vbmUiLz4KPHBh + dGggZD0iTTg4OC4yMDcgODExLjkxNEM4OTYuOTIyIDgxMS45MTQgOTA0LjY5OSA4MTMuNjI0IDkx + MS41NCA4MTcuMDQ0QzkxOC4zODEgODIwLjQ2NCA5MjMuNzgzIDgyNS43MzcgOTI3Ljc0NiA4MzIu + ODYyQzkzMS43MDkgODM5Ljk4NyA5MzMuNjkgODQ5LjE0IDkzMy42OSA4NjAuMzIxTDkzMy42OSA5 + MjIuOTg3TDkwMS45MjggOTIyLjk4N0w5MDEuOTI4IDg2NS4wNjhDOTAxLjkyOCA4NTYuMjYyIDg5 + OS45OSA4NDkuNzY1IDg5Ni4xMTMgODQ1LjU3OEM4OTIuMjM2IDg0MS4zOSA4ODYuODAxIDgzOS4y + OTYgODc5LjgwOCA4MzkuMjk2Qzg3NC43MiA4MzkuMjk2IDg3MC4xNzcgODQwLjM1NiA4NjYuMTgg + ODQyLjQ3N0M4NjIuMTgyIDg0NC41OTggODU5LjA3NSA4NDcuODIgODU2Ljg1OCA4NTIuMTQyQzg1 + NC42NDIgODU2LjQ2NSA4NTMuNTM0IDg2Mi4wMjMgODUzLjUzNCA4NjguODE5TDg1My41MzQgOTIy + Ljk4N0w4MjEuNzcyIDkyMi45ODdMODIxLjc3MiA4MTMuNDU3TDg1Mi4wMzIgODEzLjQ1N0w4NTIu + MDMyIDg0My44MjNMODQ2LjM0NyA4MzQuNzAyQzg1MC4yODEgODI3LjI3MSA4NTUuOTA3IDgyMS42 + MTUgODYzLjIyMyA4MTcuNzM1Qzg3MC41MzkgODEzLjg1NSA4NzguODY3IDgxMS45MTQgODg4LjIw + NyA4MTEuOTE0WiIgZmlsbD0iIzk5OTk5OSIgZmlsbC1ydWxlPSJub256ZXJvIiBvcGFjaXR5PSIx + IiBzdHJva2U9Im5vbmUiLz4KPC9nPgo8L2c+Cjwvc3ZnPgo= + mediatype: image/svg+xml install: spec: clusterPermissions: - - rules: - - apiGroups: - - apps - resources: - - daemonsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - - apiGroups: - - bpfman.io - resources: - - bpfprograms - verbs: - - get - - list - - watch - - apiGroups: - - bpfman.io - resources: - - configmaps/finalizers - verbs: - - update - - apiGroups: - - bpfman.io - resources: - - fentryprograms - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - - apiGroups: - - bpfman.io - resources: - - fentryprograms/finalizers - verbs: - - update - - apiGroups: - - bpfman.io - resources: - - fentryprograms/status - verbs: - - get - - patch - - update - - apiGroups: - - bpfman.io - resources: - - fexitprograms - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - - apiGroups: - - bpfman.io - resources: - - fexitprograms/finalizers - verbs: - - update - - apiGroups: - - bpfman.io - resources: - - fexitprograms/status - verbs: - - get - - patch - - update - - apiGroups: - - bpfman.io - resources: - - kprobeprograms - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - - apiGroups: - - bpfman.io - resources: - - kprobeprograms/finalizers - verbs: - - update - - apiGroups: - - bpfman.io - resources: - - kprobeprograms/status - verbs: - - get - - patch - - update - - apiGroups: - - bpfman.io - resources: - - tcprograms - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - - apiGroups: - - bpfman.io - resources: - - tcprograms/finalizers - verbs: - - update - - apiGroups: - - bpfman.io - resources: - - tcprograms/status - verbs: - - get - - patch - - update - - apiGroups: - - bpfman.io - resources: - - tracepointprograms - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - - apiGroups: - - bpfman.io - resources: - - tracepointprograms/finalizers - verbs: - - update - - apiGroups: - - bpfman.io - resources: - - tracepointprograms/status - verbs: - - get - - patch - - update - - apiGroups: - - bpfman.io - resources: - - uprobeprograms - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - - apiGroups: - - bpfman.io - resources: - - uprobeprograms/finalizers - verbs: - - update - - apiGroups: - - bpfman.io - resources: - - uprobeprograms/status - verbs: - - get - - patch - - update - - apiGroups: - - bpfman.io - resources: - - xdpprograms - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - - apiGroups: - - bpfman.io - resources: - - xdpprograms/finalizers - verbs: - - update - - apiGroups: - - bpfman.io - resources: - - xdpprograms/status - verbs: - - get - - patch - - update - - apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - get - - list - - watch - - apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch - - apiGroups: - - security.openshift.io - resources: - - securitycontextconstraints - verbs: - - create - - delete - - get - - list - - watch - - apiGroups: - - storage.k8s.io - resources: - - csidrivers - verbs: - - create - - delete - - get - - list - - watch - - apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create - - apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create - serviceAccountName: bpfman-operator + - rules: + - apiGroups: + - apps + resources: + - daemonsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - bpfman.io + resources: + - bpfprograms + verbs: + - get + - list + - watch + - apiGroups: + - bpfman.io + resources: + - configmaps/finalizers + verbs: + - update + - apiGroups: + - bpfman.io + resources: + - fentryprograms + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - bpfman.io + resources: + - fentryprograms/finalizers + verbs: + - update + - apiGroups: + - bpfman.io + resources: + - fentryprograms/status + verbs: + - get + - patch + - update + - apiGroups: + - bpfman.io + resources: + - fexitprograms + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - bpfman.io + resources: + - fexitprograms/finalizers + verbs: + - update + - apiGroups: + - bpfman.io + resources: + - fexitprograms/status + verbs: + - get + - patch + - update + - apiGroups: + - bpfman.io + resources: + - kprobeprograms + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - bpfman.io + resources: + - kprobeprograms/finalizers + verbs: + - update + - apiGroups: + - bpfman.io + resources: + - kprobeprograms/status + verbs: + - get + - patch + - update + - apiGroups: + - bpfman.io + resources: + - tcprograms + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - bpfman.io + resources: + - tcprograms/finalizers + verbs: + - update + - apiGroups: + - bpfman.io + resources: + - tcprograms/status + verbs: + - get + - patch + - update + - apiGroups: + - bpfman.io + resources: + - tracepointprograms + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - bpfman.io + resources: + - tracepointprograms/finalizers + verbs: + - update + - apiGroups: + - bpfman.io + resources: + - tracepointprograms/status + verbs: + - get + - patch + - update + - apiGroups: + - bpfman.io + resources: + - uprobeprograms + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - bpfman.io + resources: + - uprobeprograms/finalizers + verbs: + - update + - apiGroups: + - bpfman.io + resources: + - uprobeprograms/status + verbs: + - get + - patch + - update + - apiGroups: + - bpfman.io + resources: + - xdpprograms + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - bpfman.io + resources: + - xdpprograms/finalizers + verbs: + - update + - apiGroups: + - bpfman.io + resources: + - xdpprograms/status + verbs: + - get + - patch + - update + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - get + - list + - watch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + verbs: + - create + - delete + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - csidrivers + verbs: + - create + - delete + - get + - list + - watch + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create + serviceAccountName: bpfman-operator deployments: - - label: - app.kubernetes.io/component: manager - app.kubernetes.io/created-by: bpfman-operator - app.kubernetes.io/instance: controller-manager - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: deployment - app.kubernetes.io/part-of: bpfman-operator - control-plane: controller-manager - name: bpfman-operator - spec: - replicas: 1 - selector: - matchLabels: - control-plane: controller-manager - strategy: {} - template: - metadata: - annotations: - kubectl.kubernetes.io/default-container: manager - labels: + - label: + app.kubernetes.io/component: manager + app.kubernetes.io/created-by: bpfman-operator + app.kubernetes.io/instance: controller-manager + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: deployment + app.kubernetes.io/part-of: bpfman-operator + control-plane: controller-manager + name: bpfman-operator + spec: + replicas: 1 + selector: + matchLabels: control-plane: controller-manager - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/arch - operator: In - values: - - amd64 - - arm64 - - ppc64le - - s390x - - key: kubernetes.io/os - operator: In - values: - - linux - containers: - - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8174/ - - --logtostderr=true - - --v=0 - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.0 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi + strategy: {} + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + labels: + control-plane: controller-manager + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - arm64 + - ppc64le + - s390x + - key: kubernetes.io/os + operator: In + values: + - linux + containers: + - args: + - --secure-listen-address=0.0.0.0:8443 + - --upstream=http://127.0.0.1:8174/ + - --logtostderr=true + - --v=0 + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.0 + name: kube-rbac-proxy + ports: + - containerPort: 8443 + name: https + protocol: TCP + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 5m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + - args: + - --health-probe-bind-address=:8175 + - --metrics-bind-address=127.0.0.1:8174 + - --leader-elect + command: + - /bpfman-operator + env: + - name: GO_LOG + value: debug + image: quay.io/bpfman/bpfman-operator:latest + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /healthz + port: 8175 + initialDelaySeconds: 15 + periodSeconds: 20 + name: bpfman-operator + readinessProbe: + httpGet: + path: /readyz + port: 8175 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 10m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - - args: - - --health-probe-bind-address=:8175 - - --metrics-bind-address=127.0.0.1:8174 - - --leader-elect - command: - - /bpfman-operator - env: - - name: GO_LOG - value: debug - image: quay.io/bpfman/bpfman-operator:latest - imagePullPolicy: IfNotPresent - livenessProbe: - httpGet: - path: /healthz - port: 8175 - initialDelaySeconds: 15 - periodSeconds: 20 - name: bpfman-operator - readinessProbe: - httpGet: - path: /readyz - port: 8175 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 10m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - securityContext: - runAsNonRoot: true - serviceAccountName: bpfman-operator - terminationGracePeriodSeconds: 10 + runAsNonRoot: true + serviceAccountName: bpfman-operator + terminationGracePeriodSeconds: 10 permissions: - - rules: - - apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - serviceAccountName: bpfman-operator + - rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + serviceAccountName: bpfman-operator strategy: deployment installModes: - - supported: false - type: OwnNamespace - - supported: false - type: SingleNamespace - - supported: false - type: MultiNamespace - - supported: true - type: AllNamespaces + - supported: false + type: OwnNamespace + - supported: false + type: SingleNamespace + - supported: false + type: MultiNamespace + - supported: true + type: AllNamespaces keywords: - - ebpf - - kubernetes + - ebpf + - kubernetes links: - - name: bpfman website - url: https://bpfman.io/ + - name: bpfman website + url: https://bpfman.io/ maintainers: - - email: astoycos@redhat.com - name: Andrew Stoycos + - email: astoycos@redhat.com + name: Andrew Stoycos maturity: alpha provider: name: The bpfman Community diff --git a/cmd/bpfman-operator/main.go b/cmd/bpfman-operator/main.go index ba1e8dc93..6d2f3323b 100644 --- a/cmd/bpfman-operator/main.go +++ b/cmd/bpfman-operator/main.go @@ -231,6 +231,12 @@ func main() { os.Exit(1) } + if err = (&bpfmanoperator.BpfApplicationReconciler{ + ReconcilerCommon: common, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "BpfApplication") + os.Exit(1) + } //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { diff --git a/config/crd/bases/bpfman.io_bpfapplications.yaml b/config/crd/bases/bpfman.io_bpfapplications.yaml new file mode 100644 index 000000000..295294ae7 --- /dev/null +++ b/config/crd/bases/bpfman.io_bpfapplications.yaml @@ -0,0 +1,1548 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: bpfapplications.bpfman.io +spec: + group: bpfman.io + names: + kind: BpfApplication + listKind: BpfApplicationList + plural: bpfapplications + singular: bpfapplication + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: BpfApplication is the Schema for the bpfapplications API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: BpfApplicationSpec defines the desired state of BpfApplication + properties: + globaldata: + additionalProperties: + format: byte + type: string + description: |- + GlobalData allows the user to set global variables when the program is loaded + with an array of raw bytes. This is a very low level primitive. The caller + is responsible for formatting the byte string appropriately considering + such things as size, endianness, alignment and packing of data structures. + type: object + nodeselector: + description: |- + NodeSelector allows the user to specify which nodes to deploy the + bpf program to. This field must be specified, to select all nodes + use standard metav1.LabelSelector semantics and make it empty. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + programs: + description: |- + Programs is a list of bpf programs supported for a specific application. + It's possible that the application can selectively choose which program(s) + to run from this list. + items: + description: BpfApplicationProgram defines the desired state of + BpfApplication + properties: + fentry: + description: fentry defines the desired state of the application's + FentryPrograms. + properties: + bpffunctionname: + description: |- + BpfFunctionName is the name of the function that is the entry point for the BPF + program + type: string + bytecode: + description: |- + Bytecode configures where the bpf program's bytecode should be loaded + from. + properties: + image: + description: Image used to specify a bytecode container + image. + properties: + imagepullpolicy: + default: IfNotPresent + description: PullPolicy describes a policy for if/when + to pull a bytecode image. Defaults to IfNotPresent. + enum: + - Always + - Never + - IfNotPresent + type: string + imagepullsecret: + description: |- + ImagePullSecret is the name of the secret bpfman should use to get remote image + repository secrets. + properties: + name: + description: Name of the secret which contains + the credentials to access the image repository. + type: string + namespace: + description: Namespace of the secret which contains + the credentials to access the image repository. + type: string + required: + - name + - namespace + type: object + url: + description: Valid container image URL used to reference + a remote bytecode image. + type: string + required: + - url + type: object + path: + description: Path is used to specify a bytecode object + via filepath. + type: string + type: object + func_name: + description: Function to attach the fentry to. + type: string + mapownerselector: + description: |- + MapOwnerSelector is used to select the loaded eBPF program this eBPF program + will share a map with. The value is a label applied to the BpfProgram to select. + The selector must resolve to exactly one instance of a BpfProgram on a given node + or the eBPF program will not load. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - bpffunctionname + - bytecode + - func_name + type: object + fexit: + description: fexit defines the desired state of the application's + FexitPrograms. + properties: + bpffunctionname: + description: |- + BpfFunctionName is the name of the function that is the entry point for the BPF + program + type: string + bytecode: + description: |- + Bytecode configures where the bpf program's bytecode should be loaded + from. + properties: + image: + description: Image used to specify a bytecode container + image. + properties: + imagepullpolicy: + default: IfNotPresent + description: PullPolicy describes a policy for if/when + to pull a bytecode image. Defaults to IfNotPresent. + enum: + - Always + - Never + - IfNotPresent + type: string + imagepullsecret: + description: |- + ImagePullSecret is the name of the secret bpfman should use to get remote image + repository secrets. + properties: + name: + description: Name of the secret which contains + the credentials to access the image repository. + type: string + namespace: + description: Namespace of the secret which contains + the credentials to access the image repository. + type: string + required: + - name + - namespace + type: object + url: + description: Valid container image URL used to reference + a remote bytecode image. + type: string + required: + - url + type: object + path: + description: Path is used to specify a bytecode object + via filepath. + type: string + type: object + func_name: + description: Function to attach the fexit to. + type: string + mapownerselector: + description: |- + MapOwnerSelector is used to select the loaded eBPF program this eBPF program + will share a map with. The value is a label applied to the BpfProgram to select. + The selector must resolve to exactly one instance of a BpfProgram on a given node + or the eBPF program will not load. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - bpffunctionname + - bytecode + - func_name + type: object + kprobe: + description: kprobe defines the desired state of the application's + KprobePrograms. + properties: + bpffunctionname: + description: |- + BpfFunctionName is the name of the function that is the entry point for the BPF + program + type: string + bytecode: + description: |- + Bytecode configures where the bpf program's bytecode should be loaded + from. + properties: + image: + description: Image used to specify a bytecode container + image. + properties: + imagepullpolicy: + default: IfNotPresent + description: PullPolicy describes a policy for if/when + to pull a bytecode image. Defaults to IfNotPresent. + enum: + - Always + - Never + - IfNotPresent + type: string + imagepullsecret: + description: |- + ImagePullSecret is the name of the secret bpfman should use to get remote image + repository secrets. + properties: + name: + description: Name of the secret which contains + the credentials to access the image repository. + type: string + namespace: + description: Namespace of the secret which contains + the credentials to access the image repository. + type: string + required: + - name + - namespace + type: object + url: + description: Valid container image URL used to reference + a remote bytecode image. + type: string + required: + - url + type: object + path: + description: Path is used to specify a bytecode object + via filepath. + type: string + type: object + func_name: + description: Functions to attach the kprobe to. + type: string + mapownerselector: + description: |- + MapOwnerSelector is used to select the loaded eBPF program this eBPF program + will share a map with. The value is a label applied to the BpfProgram to select. + The selector must resolve to exactly one instance of a BpfProgram on a given node + or the eBPF program will not load. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + offset: + default: 0 + description: |- + Offset added to the address of the function for kprobe. + Not allowed for kretprobes. + format: int64 + type: integer + retprobe: + default: false + description: Whether the program is a kretprobe. Default + is false + type: boolean + required: + - bpffunctionname + - bytecode + - func_name + type: object + kretprobe: + description: kretprobe defines the desired state of the application's + KretprobePrograms. + properties: + bpffunctionname: + description: |- + BpfFunctionName is the name of the function that is the entry point for the BPF + program + type: string + bytecode: + description: |- + Bytecode configures where the bpf program's bytecode should be loaded + from. + properties: + image: + description: Image used to specify a bytecode container + image. + properties: + imagepullpolicy: + default: IfNotPresent + description: PullPolicy describes a policy for if/when + to pull a bytecode image. Defaults to IfNotPresent. + enum: + - Always + - Never + - IfNotPresent + type: string + imagepullsecret: + description: |- + ImagePullSecret is the name of the secret bpfman should use to get remote image + repository secrets. + properties: + name: + description: Name of the secret which contains + the credentials to access the image repository. + type: string + namespace: + description: Namespace of the secret which contains + the credentials to access the image repository. + type: string + required: + - name + - namespace + type: object + url: + description: Valid container image URL used to reference + a remote bytecode image. + type: string + required: + - url + type: object + path: + description: Path is used to specify a bytecode object + via filepath. + type: string + type: object + func_name: + description: Functions to attach the kprobe to. + type: string + mapownerselector: + description: |- + MapOwnerSelector is used to select the loaded eBPF program this eBPF program + will share a map with. The value is a label applied to the BpfProgram to select. + The selector must resolve to exactly one instance of a BpfProgram on a given node + or the eBPF program will not load. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + offset: + default: 0 + description: |- + Offset added to the address of the function for kprobe. + Not allowed for kretprobes. + format: int64 + type: integer + retprobe: + default: false + description: Whether the program is a kretprobe. Default + is false + type: boolean + required: + - bpffunctionname + - bytecode + - func_name + type: object + tc: + description: tc defines the desired state of the application's + TcPrograms. + properties: + bpffunctionname: + description: |- + BpfFunctionName is the name of the function that is the entry point for the BPF + program + type: string + bytecode: + description: |- + Bytecode configures where the bpf program's bytecode should be loaded + from. + properties: + image: + description: Image used to specify a bytecode container + image. + properties: + imagepullpolicy: + default: IfNotPresent + description: PullPolicy describes a policy for if/when + to pull a bytecode image. Defaults to IfNotPresent. + enum: + - Always + - Never + - IfNotPresent + type: string + imagepullsecret: + description: |- + ImagePullSecret is the name of the secret bpfman should use to get remote image + repository secrets. + properties: + name: + description: Name of the secret which contains + the credentials to access the image repository. + type: string + namespace: + description: Namespace of the secret which contains + the credentials to access the image repository. + type: string + required: + - name + - namespace + type: object + url: + description: Valid container image URL used to reference + a remote bytecode image. + type: string + required: + - url + type: object + path: + description: Path is used to specify a bytecode object + via filepath. + type: string + type: object + direction: + description: |- + Direction specifies the direction of traffic the tc program should + attach to for a given network device. + enum: + - ingress + - egress + type: string + interfaceselector: + description: Selector to determine the network interface + (or interfaces) + maxProperties: 1 + minProperties: 1 + properties: + interfaces: + description: |- + Interfaces refers to a list of network interfaces to attach the BPF + program to. + items: + type: string + type: array + primarynodeinterface: + description: Attach BPF program to the primary interface + on the node. Only 'true' accepted. + type: boolean + type: object + mapownerselector: + description: |- + MapOwnerSelector is used to select the loaded eBPF program this eBPF program + will share a map with. The value is a label applied to the BpfProgram to select. + The selector must resolve to exactly one instance of a BpfProgram on a given node + or the eBPF program will not load. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + priority: + description: |- + Priority specifies the priority of the tc program in relation to + other programs of the same type with the same attach point. It is a value + from 0 to 1000 where lower values have higher precedence. + format: int32 + maximum: 1000 + minimum: 0 + type: integer + proceedon: + default: + - pipe + - dispatcher_return + description: |- + ProceedOn allows the user to call other tc programs in chain on this exit code. + Multiple values are supported by repeating the parameter. + items: + enum: + - unspec + - ok + - reclassify + - shot + - pipe + - stolen + - queued + - repeat + - redirect + - trap + - dispatcher_return + type: string + maxItems: 11 + type: array + required: + - bpffunctionname + - bytecode + - direction + - interfaceselector + - priority + type: object + tracepoint: + description: tracepoint defines the desired state of the application's + TracepointPrograms. + properties: + bpffunctionname: + description: |- + BpfFunctionName is the name of the function that is the entry point for the BPF + program + type: string + bytecode: + description: |- + Bytecode configures where the bpf program's bytecode should be loaded + from. + properties: + image: + description: Image used to specify a bytecode container + image. + properties: + imagepullpolicy: + default: IfNotPresent + description: PullPolicy describes a policy for if/when + to pull a bytecode image. Defaults to IfNotPresent. + enum: + - Always + - Never + - IfNotPresent + type: string + imagepullsecret: + description: |- + ImagePullSecret is the name of the secret bpfman should use to get remote image + repository secrets. + properties: + name: + description: Name of the secret which contains + the credentials to access the image repository. + type: string + namespace: + description: Namespace of the secret which contains + the credentials to access the image repository. + type: string + required: + - name + - namespace + type: object + url: + description: Valid container image URL used to reference + a remote bytecode image. + type: string + required: + - url + type: object + path: + description: Path is used to specify a bytecode object + via filepath. + type: string + type: object + mapownerselector: + description: |- + MapOwnerSelector is used to select the loaded eBPF program this eBPF program + will share a map with. The value is a label applied to the BpfProgram to select. + The selector must resolve to exactly one instance of a BpfProgram on a given node + or the eBPF program will not load. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + names: + description: |- + Names refers to the names of kernel tracepoints to attach the + bpf program to. + items: + type: string + type: array + required: + - bpffunctionname + - bytecode + - names + type: object + type: + description: Type specifies the bpf program type + enum: + - XDP + - TC + - TCX + - Fentry + - Fexit + - Kprobe + - Kretprobe + - Uprobe + - Uretprobe + - Tracepoint + type: string + uprobe: + description: uprobe defines the desired state of the application's + UprobePrograms. + properties: + bpffunctionname: + description: |- + BpfFunctionName is the name of the function that is the entry point for the BPF + program + type: string + bytecode: + description: |- + Bytecode configures where the bpf program's bytecode should be loaded + from. + properties: + image: + description: Image used to specify a bytecode container + image. + properties: + imagepullpolicy: + default: IfNotPresent + description: PullPolicy describes a policy for if/when + to pull a bytecode image. Defaults to IfNotPresent. + enum: + - Always + - Never + - IfNotPresent + type: string + imagepullsecret: + description: |- + ImagePullSecret is the name of the secret bpfman should use to get remote image + repository secrets. + properties: + name: + description: Name of the secret which contains + the credentials to access the image repository. + type: string + namespace: + description: Namespace of the secret which contains + the credentials to access the image repository. + type: string + required: + - name + - namespace + type: object + url: + description: Valid container image URL used to reference + a remote bytecode image. + type: string + required: + - url + type: object + path: + description: Path is used to specify a bytecode object + via filepath. + type: string + type: object + containers: + description: |- + Containers identifes the set of containers in which to attach the uprobe. + If Containers is not specified, the uprobe will be attached in the + bpfman-agent container. The ContainerSelector is very flexible and even + allows the selection of all containers in a cluster. If an attempt is + made to attach uprobes to too many containers, it can have a negative + impact on on the cluster. + properties: + containernames: + description: |- + Name(s) of container(s). If none are specified, all containers in the + pod are selected. + items: + type: string + type: array + namespace: + default: "" + description: Target namespaces. + type: string + pods: + description: |- + Target pods. This field must be specified, to select all pods use + standard metav1.LabelSelector semantics and make it empty. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - pods + type: object + func_name: + description: Function to attach the uprobe to. + type: string + mapownerselector: + description: |- + MapOwnerSelector is used to select the loaded eBPF program this eBPF program + will share a map with. The value is a label applied to the BpfProgram to select. + The selector must resolve to exactly one instance of a BpfProgram on a given node + or the eBPF program will not load. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + offset: + default: 0 + description: Offset added to the address of the function + for uprobe. + format: int64 + type: integer + pid: + description: |- + Only execute uprobe for given process identification number (PID). If PID + is not provided, uprobe executes for all PIDs. + format: int32 + type: integer + retprobe: + default: false + description: Whether the program is a uretprobe. Default + is false + type: boolean + target: + description: Library name or the absolute path to a binary + or library. + type: string + required: + - bpffunctionname + - bytecode + - target + type: object + uretprobe: + description: uretprobe defines the desired state of the application's + UretprobePrograms. + properties: + bpffunctionname: + description: |- + BpfFunctionName is the name of the function that is the entry point for the BPF + program + type: string + bytecode: + description: |- + Bytecode configures where the bpf program's bytecode should be loaded + from. + properties: + image: + description: Image used to specify a bytecode container + image. + properties: + imagepullpolicy: + default: IfNotPresent + description: PullPolicy describes a policy for if/when + to pull a bytecode image. Defaults to IfNotPresent. + enum: + - Always + - Never + - IfNotPresent + type: string + imagepullsecret: + description: |- + ImagePullSecret is the name of the secret bpfman should use to get remote image + repository secrets. + properties: + name: + description: Name of the secret which contains + the credentials to access the image repository. + type: string + namespace: + description: Namespace of the secret which contains + the credentials to access the image repository. + type: string + required: + - name + - namespace + type: object + url: + description: Valid container image URL used to reference + a remote bytecode image. + type: string + required: + - url + type: object + path: + description: Path is used to specify a bytecode object + via filepath. + type: string + type: object + containers: + description: |- + Containers identifes the set of containers in which to attach the uprobe. + If Containers is not specified, the uprobe will be attached in the + bpfman-agent container. The ContainerSelector is very flexible and even + allows the selection of all containers in a cluster. If an attempt is + made to attach uprobes to too many containers, it can have a negative + impact on on the cluster. + properties: + containernames: + description: |- + Name(s) of container(s). If none are specified, all containers in the + pod are selected. + items: + type: string + type: array + namespace: + default: "" + description: Target namespaces. + type: string + pods: + description: |- + Target pods. This field must be specified, to select all pods use + standard metav1.LabelSelector semantics and make it empty. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - pods + type: object + func_name: + description: Function to attach the uprobe to. + type: string + mapownerselector: + description: |- + MapOwnerSelector is used to select the loaded eBPF program this eBPF program + will share a map with. The value is a label applied to the BpfProgram to select. + The selector must resolve to exactly one instance of a BpfProgram on a given node + or the eBPF program will not load. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + offset: + default: 0 + description: Offset added to the address of the function + for uprobe. + format: int64 + type: integer + pid: + description: |- + Only execute uprobe for given process identification number (PID). If PID + is not provided, uprobe executes for all PIDs. + format: int32 + type: integer + retprobe: + default: false + description: Whether the program is a uretprobe. Default + is false + type: boolean + target: + description: Library name or the absolute path to a binary + or library. + type: string + required: + - bpffunctionname + - bytecode + - target + type: object + xdp: + description: xdp defines the desired state of the application's + XdpPrograms. + properties: + bpffunctionname: + description: |- + BpfFunctionName is the name of the function that is the entry point for the BPF + program + type: string + bytecode: + description: |- + Bytecode configures where the bpf program's bytecode should be loaded + from. + properties: + image: + description: Image used to specify a bytecode container + image. + properties: + imagepullpolicy: + default: IfNotPresent + description: PullPolicy describes a policy for if/when + to pull a bytecode image. Defaults to IfNotPresent. + enum: + - Always + - Never + - IfNotPresent + type: string + imagepullsecret: + description: |- + ImagePullSecret is the name of the secret bpfman should use to get remote image + repository secrets. + properties: + name: + description: Name of the secret which contains + the credentials to access the image repository. + type: string + namespace: + description: Namespace of the secret which contains + the credentials to access the image repository. + type: string + required: + - name + - namespace + type: object + url: + description: Valid container image URL used to reference + a remote bytecode image. + type: string + required: + - url + type: object + path: + description: Path is used to specify a bytecode object + via filepath. + type: string + type: object + interfaceselector: + description: Selector to determine the network interface + (or interfaces) + maxProperties: 1 + minProperties: 1 + properties: + interfaces: + description: |- + Interfaces refers to a list of network interfaces to attach the BPF + program to. + items: + type: string + type: array + primarynodeinterface: + description: Attach BPF program to the primary interface + on the node. Only 'true' accepted. + type: boolean + type: object + mapownerselector: + description: |- + MapOwnerSelector is used to select the loaded eBPF program this eBPF program + will share a map with. The value is a label applied to the BpfProgram to select. + The selector must resolve to exactly one instance of a BpfProgram on a given node + or the eBPF program will not load. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + priority: + description: |- + Priority specifies the priority of the bpf program in relation to + other programs of the same type with the same attach point. It is a value + from 0 to 1000 where lower values have higher precedence. + format: int32 + maximum: 1000 + minimum: 0 + type: integer + proceedon: + default: + - pass + - dispatcher_return + items: + enum: + - aborted + - drop + - pass + - tx + - redirect + - dispatcher_return + type: string + maxItems: 6 + type: array + required: + - bpffunctionname + - bytecode + - interfaceselector + - priority + type: object + type: object + minItems: 1 + type: array + required: + - nodeselector + type: object + status: + description: BpfApplicationStatus defines the observed state of BpfApplication + properties: + conditions: + description: |- + Conditions houses the global cluster state for the eBPFProgram. The explicit + condition types are defined internally. + items: + description: "Condition contains details for one aspect of the current + state of this API Resource.\n---\nThis struct is intended for + direct use as an array at the field path .status.conditions. For + example,\n\n\n\ttype FooStatus struct{\n\t // Represents the + observations of a foo's current state.\n\t // Known .status.conditions.type + are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // + +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t + \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t + \ // other fields\n\t}" + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index ba17a6bcc..ae82d432b 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -10,7 +10,7 @@ resources: - bases/bpfman.io_uprobeprograms.yaml - bases/bpfman.io_fentryprograms.yaml - bases/bpfman.io_fexitprograms.yaml - + - bases/bpfman.io_bpfapplications.yaml #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: @@ -24,6 +24,7 @@ patchesStrategicMerge: #- patches/webhook_in_uprobeprograms.yaml #- patches/webhook_in_fentryprograms.yaml #- patches/webhook_in_fexitprograms.yaml +#- patches/webhook_in_bpfapplications.yaml #+kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. @@ -36,6 +37,7 @@ patchesStrategicMerge: #- patches/cainjection_in_uprobeprograms.yaml #- patches/cainjection_in_fentryprograms.yaml #- patches/cainjection_in_fentryprograms.yaml +#- patches/cainjection_in_bpfapplications.yaml #+kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/config/crd/patches/cainjection_in_bpfapplications.yaml b/config/crd/patches/cainjection_in_bpfapplications.yaml new file mode 100644 index 000000000..b282ec14e --- /dev/null +++ b/config/crd/patches/cainjection_in_bpfapplications.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: bpfapplications.bpfman.io diff --git a/config/crd/patches/webhook_in_bpfapplications.yaml b/config/crd/patches/webhook_in_bpfapplications.yaml new file mode 100644 index 000000000..9c0c7f2f5 --- /dev/null +++ b/config/crd/patches/webhook_in_bpfapplications.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bpfapplications.bpfman.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/rbac/bpfapplication_editor_role.yaml b/config/rbac/bpfapplication_editor_role.yaml new file mode 100644 index 000000000..b803892b2 --- /dev/null +++ b/config/rbac/bpfapplication_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit bpfapplications. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: bpfapplication-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: bpfman-operator + app.kubernetes.io/part-of: bpfman-operator + app.kubernetes.io/managed-by: kustomize + name: bpfapplication-editor-role +rules: +- apiGroups: + - bpfman.io + resources: + - bpfapplications + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - bpfman.io + resources: + - bpfapplications/status + verbs: + - get diff --git a/config/rbac/bpfapplication_viewer_role.yaml b/config/rbac/bpfapplication_viewer_role.yaml new file mode 100644 index 000000000..81121592d --- /dev/null +++ b/config/rbac/bpfapplication_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view bpfapplications. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: bpfapplication-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: bpfman-operator + app.kubernetes.io/part-of: bpfman-operator + app.kubernetes.io/managed-by: kustomize + name: bpfapplication-viewer-role +rules: +- apiGroups: + - bpfman.io + resources: + - bpfapplications + verbs: + - get + - list + - watch +- apiGroups: + - bpfman.io + resources: + - bpfapplications/status + verbs: + - get diff --git a/config/rbac/bpfman-operator/role.yaml b/config/rbac/bpfman-operator/role.yaml index 629fc6bb9..3a6848ce5 100644 --- a/config/rbac/bpfman-operator/role.yaml +++ b/config/rbac/bpfman-operator/role.yaml @@ -16,6 +16,32 @@ rules: - patch - update - watch +- apiGroups: + - bpfman.io + resources: + - bpfapplications + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - bpfman.io + resources: + - bpfapplications/finalizers + verbs: + - update +- apiGroups: + - bpfman.io + resources: + - bpfapplications/status + verbs: + - get + - patch + - update - apiGroups: - bpfman.io resources: diff --git a/config/samples/_v1alpha1_bpfapplication.yaml b/config/samples/_v1alpha1_bpfapplication.yaml new file mode 100644 index 000000000..3bb28d1d6 --- /dev/null +++ b/config/samples/_v1alpha1_bpfapplication.yaml @@ -0,0 +1,12 @@ +apiVersion: bpfman.io/v1alpha1 +kind: BpfApplication +metadata: + labels: + app.kubernetes.io/name: bpfapplication + app.kubernetes.io/instance: bpfapplication-sample + app.kubernetes.io/part-of: bpfman-operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: bpfman-operator + name: bpfapplication-sample +spec: + # TODO(user): Add fields here diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index e6f2d9428..c6c0285af 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -7,4 +7,5 @@ resources: - bpfman.io_v1alpha1_uprobe_uprobeprogram.yaml - bpfman.io_v1alpha1_fentry_fentryprogram.yaml - bpfman.io_v1alpha1_fexit_fexitprogram.yaml +- _v1alpha1_bpfapplication.yaml # +kubebuilder:scaffold:manifestskustomizesamples diff --git a/controllers/bpfman-operator/application-programs.go b/controllers/bpfman-operator/application-programs.go new file mode 100644 index 000000000..8e9afa2ab --- /dev/null +++ b/controllers/bpfman-operator/application-programs.go @@ -0,0 +1,162 @@ +/* +Copyright 2023 The bpfman Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bpfmanoperator + +import ( + "context" + "fmt" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/log" + + bpfmaniov1alpha1 "github.com/bpfman/bpfman-operator/apis/v1alpha1" + internal "github.com/bpfman/bpfman-operator/internal" +) + +// BpfApplicationReconciler reconciles a BpfApplication object +type BpfApplicationReconciler struct { + ReconcilerCommon +} + +func (r *BpfApplicationReconciler) getRecCommon() *ReconcilerCommon { + return &r.ReconcilerCommon +} + +func (r *BpfApplicationReconciler) getFinalizer() string { + return internal.BpfApplicationControllerFinalizer +} + +//+kubebuilder:rbac:groups=bpfman.io,resources=bpfapplications,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=bpfman.io,resources=bpfapplications/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=bpfman.io,resources=bpfapplications/finalizers,verbs=update + +func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + r.Logger = log.FromContext(ctx) + + appProgram := &bpfmaniov1alpha1.BpfApplication{} + if err := r.Get(ctx, req.NamespacedName, appProgram); err != nil { + // Reconcile was triggered by bpfProgram event, get parent appProgram Object. + if errors.IsNotFound(err) { + bpfProgram := &bpfmaniov1alpha1.BpfProgram{} + if err := r.Get(ctx, req.NamespacedName, bpfProgram); err != nil { + if errors.IsNotFound(err) { + r.Logger.V(1).Info("bpfProgram not found stale reconcile, exiting", "Name", req.NamespacedName) + } else { + r.Logger.Error(err, "failed getting bpfProgram Object", "Name", req.NamespacedName) + } + return ctrl.Result{}, nil + } + + // Get owning appProgram object from ownerRef + ownerRef := metav1.GetControllerOf(bpfProgram) + if ownerRef == nil { + return ctrl.Result{Requeue: false}, fmt.Errorf("failed getting bpfProgram Object owner") + } + + if err := r.Get(ctx, types.NamespacedName{Namespace: corev1.NamespaceAll, Name: ownerRef.Name}, appProgram); err != nil { + if errors.IsNotFound(err) { + r.Logger.Info("Application Programs from ownerRef not found stale reconcile exiting", "Name", req.NamespacedName) + } else { + r.Logger.Error(err, "failed getting Application Programs Object from ownerRef", "Name", req.NamespacedName) + } + return ctrl.Result{}, nil + } + + } else { + r.Logger.Error(err, "failed getting Application Programs Object", "Name", req.NamespacedName) + return ctrl.Result{}, nil + } + } + + return r.reconcileAppPrograms(ctx, appProgram) +} + +func (r *BpfApplicationReconciler) reconcileAppPrograms(ctx context.Context, application *bpfmaniov1alpha1.BpfApplication) (ctrl.Result, error) { + var result ctrl.Result + var err error + + for _, prog := range application.Spec.Programs { + switch prog.Type { + case bpfmaniov1alpha1.ProgTypeXDP: + r.Logger.Info("Reconciling Application XDP Programs") + result, err = reconcileBpfProgram(ctx, r, prog.XDP) + + case bpfmaniov1alpha1.ProgTypeTC: + r.Logger.Info("Reconciling Application TC Programs") + result, err = reconcileBpfProgram(ctx, r, prog.TC) + + case bpfmaniov1alpha1.ProgTypeFentry: + r.Logger.Info("Reconciling Application Fentry/Fexit Programs") + result, err = reconcileBpfProgram(ctx, r, prog.Fentry) + + case bpfmaniov1alpha1.ProgTypeFexit: + r.Logger.Info("Reconciling Application Fexit Programs") + result, err = reconcileBpfProgram(ctx, r, prog.Fexit) + + case bpfmaniov1alpha1.ProgTypeKprobe: + r.Logger.Info("Reconciling Application Kprobe Programs") + result, err = reconcileBpfProgram(ctx, r, prog.Kprobe) + + case bpfmaniov1alpha1.ProgTypeKretprobe: + r.Logger.Info("Reconciling Application Kretprobe Programs") + result, err = reconcileBpfProgram(ctx, r, prog.Kretprobe) + + case bpfmaniov1alpha1.ProgTypeUprobe: + r.Logger.Info("Reconciling Application Uprobe Programs") + result, err = reconcileBpfProgram(ctx, r, prog.Uprobe) + + case bpfmaniov1alpha1.ProgTypeUretprobe: + r.Logger.Info("Reconciling Application Uretprobe Programs") + result, err = reconcileBpfProgram(ctx, r, prog.Uretprobe) + + case bpfmaniov1alpha1.ProgTypeTracepoint: + r.Logger.Info("Reconciling Application Tracepoint Programs") + result, err = reconcileBpfProgram(ctx, r, prog.Tracepoint) + + default: + err := fmt.Errorf("invalid program type: %s", prog.Type) + r.Logger.Error(err, "invalid program type") + return ctrl.Result{}, err + } + if err != nil { + r.Logger.Error(err, "failed reconciling Application Programs") + return ctrl.Result{}, err + } + } + return result, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *BpfApplicationReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&bpfmaniov1alpha1.BpfApplication{}). + Complete(r) +} + +func (r *BpfApplicationReconciler) updateStatus(ctx context.Context, name string, cond bpfmaniov1alpha1.ProgramConditionType, message string) (ctrl.Result, error) { + app := &bpfmaniov1alpha1.BpfApplication{} + if err := r.Get(ctx, types.NamespacedName{Namespace: corev1.NamespaceAll, Name: name}, app); err != nil { + r.Logger.V(1).Info("failed to get fresh Application Programs object...requeuing") + return ctrl.Result{Requeue: true, RequeueAfter: retryDurationOperator}, nil + } + + return r.updateCondition(ctx, app, &app.Status.Conditions, cond, message) +} diff --git a/internal/constants.go b/internal/constants.go index ae9b40191..473108805 100644 --- a/internal/constants.go +++ b/internal/constants.go @@ -81,6 +81,8 @@ const ( // FexitProgramControllerFinalizer is the finalizer that holds a Fexit // BpfProgram object from deletion until cleanup can be performed. FexitProgramControllerFinalizer = "bpfman.io.fexitprogramcontroller/finalizer" + // BpfApplicationFinalizer is the finalizer that holds a BpfApplication + BpfApplicationControllerFinalizer = "bpfman.io.bpfapplicationcontroller/finalizer" ) // Must match the kernel's `bpf_prog_type` enum. diff --git a/pkg/client/apis/v1alpha1/bpfapplication.go b/pkg/client/apis/v1alpha1/bpfapplication.go new file mode 100644 index 000000000..3c154a5ef --- /dev/null +++ b/pkg/client/apis/v1alpha1/bpfapplication.go @@ -0,0 +1,68 @@ +/* +Copyright 2023 The bpfman Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + v1alpha1 "github.com/bpfman/bpfman-operator/apis/v1alpha1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/cache" +) + +// BpfApplicationLister helps list BpfApplications. +// All objects returned here must be treated as read-only. +type BpfApplicationLister interface { + // List lists all BpfApplications in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1alpha1.BpfApplication, err error) + // Get retrieves the BpfApplication from the index for a given name. + // Objects returned here must be treated as read-only. + Get(name string) (*v1alpha1.BpfApplication, error) + BpfApplicationListerExpansion +} + +// bpfApplicationLister implements the BpfApplicationLister interface. +type bpfApplicationLister struct { + indexer cache.Indexer +} + +// NewBpfApplicationLister returns a new BpfApplicationLister. +func NewBpfApplicationLister(indexer cache.Indexer) BpfApplicationLister { + return &bpfApplicationLister{indexer: indexer} +} + +// List lists all BpfApplications in the indexer. +func (s *bpfApplicationLister) List(selector labels.Selector) (ret []*v1alpha1.BpfApplication, err error) { + err = cache.ListAll(s.indexer, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha1.BpfApplication)) + }) + return ret, err +} + +// Get retrieves the BpfApplication from the index for a given name. +func (s *bpfApplicationLister) Get(name string) (*v1alpha1.BpfApplication, error) { + obj, exists, err := s.indexer.GetByKey(name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(v1alpha1.Resource("bpfapplication"), name) + } + return obj.(*v1alpha1.BpfApplication), nil +} diff --git a/pkg/client/apis/v1alpha1/expansion_generated.go b/pkg/client/apis/v1alpha1/expansion_generated.go index 5ee0564f6..7c31cea1b 100644 --- a/pkg/client/apis/v1alpha1/expansion_generated.go +++ b/pkg/client/apis/v1alpha1/expansion_generated.go @@ -18,6 +18,10 @@ limitations under the License. package v1alpha1 +// BpfApplicationListerExpansion allows custom methods to be added to +// BpfApplicationLister. +type BpfApplicationListerExpansion interface{} + // BpfProgramListerExpansion allows custom methods to be added to // BpfProgramLister. type BpfProgramListerExpansion interface{} diff --git a/pkg/client/clientset/typed/apis/v1alpha1/apis_client.go b/pkg/client/clientset/typed/apis/v1alpha1/apis_client.go index 296b4cf3a..c1d54ecdd 100644 --- a/pkg/client/clientset/typed/apis/v1alpha1/apis_client.go +++ b/pkg/client/clientset/typed/apis/v1alpha1/apis_client.go @@ -28,6 +28,7 @@ import ( type BpfmanV1alpha1Interface interface { RESTClient() rest.Interface + BpfApplicationsGetter BpfProgramsGetter FentryProgramsGetter FexitProgramsGetter @@ -43,6 +44,10 @@ type BpfmanV1alpha1Client struct { restClient rest.Interface } +func (c *BpfmanV1alpha1Client) BpfApplications() BpfApplicationInterface { + return newBpfApplications(c) +} + func (c *BpfmanV1alpha1Client) BpfPrograms() BpfProgramInterface { return newBpfPrograms(c) } diff --git a/pkg/client/clientset/typed/apis/v1alpha1/bpfapplication.go b/pkg/client/clientset/typed/apis/v1alpha1/bpfapplication.go new file mode 100644 index 000000000..3c57ae534 --- /dev/null +++ b/pkg/client/clientset/typed/apis/v1alpha1/bpfapplication.go @@ -0,0 +1,184 @@ +/* +Copyright 2023 The bpfman Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + "time" + + v1alpha1 "github.com/bpfman/bpfman-operator/apis/v1alpha1" + scheme "github.com/bpfman/bpfman-operator/pkg/client/clientset/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// BpfApplicationsGetter has a method to return a BpfApplicationInterface. +// A group's client should implement this interface. +type BpfApplicationsGetter interface { + BpfApplications() BpfApplicationInterface +} + +// BpfApplicationInterface has methods to work with BpfApplication resources. +type BpfApplicationInterface interface { + Create(ctx context.Context, bpfApplication *v1alpha1.BpfApplication, opts v1.CreateOptions) (*v1alpha1.BpfApplication, error) + Update(ctx context.Context, bpfApplication *v1alpha1.BpfApplication, opts v1.UpdateOptions) (*v1alpha1.BpfApplication, error) + UpdateStatus(ctx context.Context, bpfApplication *v1alpha1.BpfApplication, opts v1.UpdateOptions) (*v1alpha1.BpfApplication, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.BpfApplication, error) + List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.BpfApplicationList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.BpfApplication, err error) + BpfApplicationExpansion +} + +// bpfApplications implements BpfApplicationInterface +type bpfApplications struct { + client rest.Interface +} + +// newBpfApplications returns a BpfApplications +func newBpfApplications(c *BpfmanV1alpha1Client) *bpfApplications { + return &bpfApplications{ + client: c.RESTClient(), + } +} + +// Get takes name of the bpfApplication, and returns the corresponding bpfApplication object, and an error if there is any. +func (c *bpfApplications) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.BpfApplication, err error) { + result = &v1alpha1.BpfApplication{} + err = c.client.Get(). + Resource("bpfapplications"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of BpfApplications that match those selectors. +func (c *bpfApplications) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.BpfApplicationList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v1alpha1.BpfApplicationList{} + err = c.client.Get(). + Resource("bpfapplications"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(ctx). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested bpfApplications. +func (c *bpfApplications) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Resource("bpfapplications"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch(ctx) +} + +// Create takes the representation of a bpfApplication and creates it. Returns the server's representation of the bpfApplication, and an error, if there is any. +func (c *bpfApplications) Create(ctx context.Context, bpfApplication *v1alpha1.BpfApplication, opts v1.CreateOptions) (result *v1alpha1.BpfApplication, err error) { + result = &v1alpha1.BpfApplication{} + err = c.client.Post(). + Resource("bpfapplications"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(bpfApplication). + Do(ctx). + Into(result) + return +} + +// Update takes the representation of a bpfApplication and updates it. Returns the server's representation of the bpfApplication, and an error, if there is any. +func (c *bpfApplications) Update(ctx context.Context, bpfApplication *v1alpha1.BpfApplication, opts v1.UpdateOptions) (result *v1alpha1.BpfApplication, err error) { + result = &v1alpha1.BpfApplication{} + err = c.client.Put(). + Resource("bpfapplications"). + Name(bpfApplication.Name). + VersionedParams(&opts, scheme.ParameterCodec). + Body(bpfApplication). + Do(ctx). + Into(result) + return +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *bpfApplications) UpdateStatus(ctx context.Context, bpfApplication *v1alpha1.BpfApplication, opts v1.UpdateOptions) (result *v1alpha1.BpfApplication, err error) { + result = &v1alpha1.BpfApplication{} + err = c.client.Put(). + Resource("bpfapplications"). + Name(bpfApplication.Name). + SubResource("status"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(bpfApplication). + Do(ctx). + Into(result) + return +} + +// Delete takes name of the bpfApplication and deletes it. Returns an error if one occurs. +func (c *bpfApplications) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + return c.client.Delete(). + Resource("bpfapplications"). + Name(name). + Body(&opts). + Do(ctx). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *bpfApplications) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + var timeout time.Duration + if listOpts.TimeoutSeconds != nil { + timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second + } + return c.client.Delete(). + Resource("bpfapplications"). + VersionedParams(&listOpts, scheme.ParameterCodec). + Timeout(timeout). + Body(&opts). + Do(ctx). + Error() +} + +// Patch applies the patch and returns the patched bpfApplication. +func (c *bpfApplications) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.BpfApplication, err error) { + result = &v1alpha1.BpfApplication{} + err = c.client.Patch(pt). + Resource("bpfapplications"). + Name(name). + SubResource(subresources...). + VersionedParams(&opts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} diff --git a/pkg/client/clientset/typed/apis/v1alpha1/fake/fake_apis_client.go b/pkg/client/clientset/typed/apis/v1alpha1/fake/fake_apis_client.go index 8d7ce4242..7acfcffc8 100644 --- a/pkg/client/clientset/typed/apis/v1alpha1/fake/fake_apis_client.go +++ b/pkg/client/clientset/typed/apis/v1alpha1/fake/fake_apis_client.go @@ -28,6 +28,10 @@ type FakeBpfmanV1alpha1 struct { *testing.Fake } +func (c *FakeBpfmanV1alpha1) BpfApplications() v1alpha1.BpfApplicationInterface { + return &FakeBpfApplications{c} +} + func (c *FakeBpfmanV1alpha1) BpfPrograms() v1alpha1.BpfProgramInterface { return &FakeBpfPrograms{c} } diff --git a/pkg/client/clientset/typed/apis/v1alpha1/fake/fake_bpfapplication.go b/pkg/client/clientset/typed/apis/v1alpha1/fake/fake_bpfapplication.go new file mode 100644 index 000000000..0b03c0179 --- /dev/null +++ b/pkg/client/clientset/typed/apis/v1alpha1/fake/fake_bpfapplication.go @@ -0,0 +1,132 @@ +/* +Copyright 2023 The bpfman Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1alpha1 "github.com/bpfman/bpfman-operator/apis/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeBpfApplications implements BpfApplicationInterface +type FakeBpfApplications struct { + Fake *FakeBpfmanV1alpha1 +} + +var bpfapplicationsResource = v1alpha1.SchemeGroupVersion.WithResource("bpfapplications") + +var bpfapplicationsKind = v1alpha1.SchemeGroupVersion.WithKind("BpfApplication") + +// Get takes name of the bpfApplication, and returns the corresponding bpfApplication object, and an error if there is any. +func (c *FakeBpfApplications) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.BpfApplication, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootGetAction(bpfapplicationsResource, name), &v1alpha1.BpfApplication{}) + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.BpfApplication), err +} + +// List takes label and field selectors, and returns the list of BpfApplications that match those selectors. +func (c *FakeBpfApplications) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.BpfApplicationList, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootListAction(bpfapplicationsResource, bpfapplicationsKind, opts), &v1alpha1.BpfApplicationList{}) + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha1.BpfApplicationList{ListMeta: obj.(*v1alpha1.BpfApplicationList).ListMeta} + for _, item := range obj.(*v1alpha1.BpfApplicationList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested bpfApplications. +func (c *FakeBpfApplications) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewRootWatchAction(bpfapplicationsResource, opts)) +} + +// Create takes the representation of a bpfApplication and creates it. Returns the server's representation of the bpfApplication, and an error, if there is any. +func (c *FakeBpfApplications) Create(ctx context.Context, bpfApplication *v1alpha1.BpfApplication, opts v1.CreateOptions) (result *v1alpha1.BpfApplication, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootCreateAction(bpfapplicationsResource, bpfApplication), &v1alpha1.BpfApplication{}) + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.BpfApplication), err +} + +// Update takes the representation of a bpfApplication and updates it. Returns the server's representation of the bpfApplication, and an error, if there is any. +func (c *FakeBpfApplications) Update(ctx context.Context, bpfApplication *v1alpha1.BpfApplication, opts v1.UpdateOptions) (result *v1alpha1.BpfApplication, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateAction(bpfapplicationsResource, bpfApplication), &v1alpha1.BpfApplication{}) + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.BpfApplication), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeBpfApplications) UpdateStatus(ctx context.Context, bpfApplication *v1alpha1.BpfApplication, opts v1.UpdateOptions) (*v1alpha1.BpfApplication, error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateSubresourceAction(bpfapplicationsResource, "status", bpfApplication), &v1alpha1.BpfApplication{}) + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.BpfApplication), err +} + +// Delete takes name of the bpfApplication and deletes it. Returns an error if one occurs. +func (c *FakeBpfApplications) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewRootDeleteActionWithOptions(bpfapplicationsResource, name, opts), &v1alpha1.BpfApplication{}) + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeBpfApplications) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewRootDeleteCollectionAction(bpfapplicationsResource, listOpts) + + _, err := c.Fake.Invokes(action, &v1alpha1.BpfApplicationList{}) + return err +} + +// Patch applies the patch and returns the patched bpfApplication. +func (c *FakeBpfApplications) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.BpfApplication, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootPatchSubresourceAction(bpfapplicationsResource, name, pt, data, subresources...), &v1alpha1.BpfApplication{}) + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.BpfApplication), err +} diff --git a/pkg/client/clientset/typed/apis/v1alpha1/generated_expansion.go b/pkg/client/clientset/typed/apis/v1alpha1/generated_expansion.go index ff5a8ed4d..6c297eeee 100644 --- a/pkg/client/clientset/typed/apis/v1alpha1/generated_expansion.go +++ b/pkg/client/clientset/typed/apis/v1alpha1/generated_expansion.go @@ -18,6 +18,8 @@ limitations under the License. package v1alpha1 +type BpfApplicationExpansion interface{} + type BpfProgramExpansion interface{} type FentryProgramExpansion interface{} diff --git a/pkg/client/externalversions/apis/v1alpha1/bpfapplication.go b/pkg/client/externalversions/apis/v1alpha1/bpfapplication.go new file mode 100644 index 000000000..e2cff2a7e --- /dev/null +++ b/pkg/client/externalversions/apis/v1alpha1/bpfapplication.go @@ -0,0 +1,89 @@ +/* +Copyright 2023 The bpfman Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + time "time" + + apisv1alpha1 "github.com/bpfman/bpfman-operator/apis/v1alpha1" + v1alpha1 "github.com/bpfman/bpfman-operator/pkg/client/apis/v1alpha1" + clientset "github.com/bpfman/bpfman-operator/pkg/client/clientset" + internalinterfaces "github.com/bpfman/bpfman-operator/pkg/client/externalversions/internalinterfaces" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// BpfApplicationInformer provides access to a shared informer and lister for +// BpfApplications. +type BpfApplicationInformer interface { + Informer() cache.SharedIndexInformer + Lister() v1alpha1.BpfApplicationLister +} + +type bpfApplicationInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// NewBpfApplicationInformer constructs a new informer for BpfApplication type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewBpfApplicationInformer(client clientset.Interface, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredBpfApplicationInformer(client, resyncPeriod, indexers, nil) +} + +// NewFilteredBpfApplicationInformer constructs a new informer for BpfApplication type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredBpfApplicationInformer(client clientset.Interface, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.BpfmanV1alpha1().BpfApplications().List(context.TODO(), options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.BpfmanV1alpha1().BpfApplications().Watch(context.TODO(), options) + }, + }, + &apisv1alpha1.BpfApplication{}, + resyncPeriod, + indexers, + ) +} + +func (f *bpfApplicationInformer) defaultInformer(client clientset.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredBpfApplicationInformer(client, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *bpfApplicationInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&apisv1alpha1.BpfApplication{}, f.defaultInformer) +} + +func (f *bpfApplicationInformer) Lister() v1alpha1.BpfApplicationLister { + return v1alpha1.NewBpfApplicationLister(f.Informer().GetIndexer()) +} diff --git a/pkg/client/externalversions/apis/v1alpha1/interface.go b/pkg/client/externalversions/apis/v1alpha1/interface.go index 362646dba..0b792e2fe 100644 --- a/pkg/client/externalversions/apis/v1alpha1/interface.go +++ b/pkg/client/externalversions/apis/v1alpha1/interface.go @@ -24,6 +24,8 @@ import ( // Interface provides access to all the informers in this group version. type Interface interface { + // BpfApplications returns a BpfApplicationInformer. + BpfApplications() BpfApplicationInformer // BpfPrograms returns a BpfProgramInformer. BpfPrograms() BpfProgramInformer // FentryPrograms returns a FentryProgramInformer. @@ -53,6 +55,11 @@ func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakList return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} } +// BpfApplications returns a BpfApplicationInformer. +func (v *version) BpfApplications() BpfApplicationInformer { + return &bpfApplicationInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} +} + // BpfPrograms returns a BpfProgramInformer. func (v *version) BpfPrograms() BpfProgramInformer { return &bpfProgramInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} diff --git a/pkg/client/externalversions/generic.go b/pkg/client/externalversions/generic.go index 85295806c..e045549cf 100644 --- a/pkg/client/externalversions/generic.go +++ b/pkg/client/externalversions/generic.go @@ -53,6 +53,8 @@ func (f *genericInformer) Lister() cache.GenericLister { func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) { switch resource { // Group=bpfman.io, Version=v1alpha1 + case v1alpha1.SchemeGroupVersion.WithResource("bpfapplications"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Bpfman().V1alpha1().BpfApplications().Informer()}, nil case v1alpha1.SchemeGroupVersion.WithResource("bpfprograms"): return &genericInformer{resource: resource.GroupResource(), informer: f.Bpfman().V1alpha1().BpfPrograms().Informer()}, nil case v1alpha1.SchemeGroupVersion.WithResource("fentryprograms"): diff --git a/pkg/helpers/helpers.go b/pkg/helpers/helpers.go index f48a152d6..62bfe22cf 100644 --- a/pkg/helpers/helpers.go +++ b/pkg/helpers/helpers.go @@ -21,21 +21,15 @@ import ( "fmt" "time" - //bpfmaniov1alpha1 "github.com/bpfman/bpfman-operator/apis/v1alpha1" + bpfmaniov1alpha1 "github.com/bpfman/bpfman-operator/apis/v1alpha1" bpfmanclientset "github.com/bpfman/bpfman-operator/pkg/client/clientset" - //"k8s.io/apimachinery/pkg/api/errors" - //"k8s.io/apimachinery/pkg/runtime" - ctrl "sigs.k8s.io/controller-runtime" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - //"k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/discovery" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" - - bpfmaniov1alpha1 "github.com/bpfman/bpfman-operator/apis/v1alpha1" + ctrl "sigs.k8s.io/controller-runtime" ) // Must match the internal bpfman-api mappings From 98e6266b4fe7257cf4aba5c8bd29b07417fe8f19 Mon Sep 17 00:00:00 2001 From: Mohamed Mahmoud Date: Tue, 4 Jun 2024 16:21:00 -0400 Subject: [PATCH 02/16] add application object controllers Signed-off-by: Mohamed Mahmoud --- cmd/bpfman-agent/main.go | 7 + .../bpfman-agent/application-program.go | 242 ++++++++++++++++++ .../bpfman-agent/application-program_test.go | 91 +++++++ .../application-program_test.go | 221 ++++++++++++++++ .../bpfman-operator/application-programs.go | 59 +---- 5 files changed, 563 insertions(+), 57 deletions(-) create mode 100644 controllers/bpfman-agent/application-program.go create mode 100644 controllers/bpfman-agent/application-program_test.go create mode 100644 controllers/bpfman-operator/application-program_test.go diff --git a/cmd/bpfman-agent/main.go b/cmd/bpfman-agent/main.go index 9d63bac18..c99630adc 100644 --- a/cmd/bpfman-agent/main.go +++ b/cmd/bpfman-agent/main.go @@ -193,6 +193,13 @@ func main() { os.Exit(1) } + if err = (&bpfmanagent.BpfApplicationReconciler{ + ReconcilerCommon: common, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create BpfApplicationProgram controller", "controller", "BpfProgram") + os.Exit(1) + } + //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { diff --git a/controllers/bpfman-agent/application-program.go b/controllers/bpfman-agent/application-program.go new file mode 100644 index 000000000..64d131afb --- /dev/null +++ b/controllers/bpfman-agent/application-program.go @@ -0,0 +1,242 @@ +package bpfmanagent + +import ( + "context" + "fmt" + + bpfmaniov1alpha1 "github.com/bpfman/bpfman-operator/apis/v1alpha1" + "github.com/bpfman/bpfman-operator/internal" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +type BpfApplicationReconciler struct { + ReconcilerCommon + currentApp *bpfmaniov1alpha1.BpfApplication + ourNode *v1.Node +} + +func (r *BpfApplicationReconciler) getFinalizer() string { + return internal.BpfApplicationControllerFinalizer +} + +func (r *BpfApplicationReconciler) getName() string { + return r.currentApp.Name +} + +func (r *BpfApplicationReconciler) getNode() *v1.Node { + return r.ourNode +} + +func (r *BpfApplicationReconciler) getNodeSelector() *metav1.LabelSelector { + return &r.currentApp.Spec.NodeSelector +} + +func (r *BpfApplicationReconciler) getBpfGlobalData() map[string][]byte { + return r.currentApp.Spec.GlobalData +} + +func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + // Initialize node and current program + r.currentApp = &bpfmaniov1alpha1.BpfApplication{} + r.ourNode = &v1.Node{} + r.Logger = ctrl.Log.WithName("application") + + ctxLogger := log.FromContext(ctx) + ctxLogger.Info("Reconcile Application: Enter", "ReconcileKey", req) + + // Lookup K8s node object for this bpfman-agent This should always succeed + if err := r.Get(ctx, types.NamespacedName{Namespace: v1.NamespaceAll, Name: r.NodeName}, r.ourNode); err != nil { + return ctrl.Result{Requeue: false}, fmt.Errorf("failed getting bpfman-agent node %s : %v", + req.NamespacedName, err) + } + + appPrograms := &bpfmaniov1alpha1.BpfApplicationList{} + + opts := []client.ListOption{} + + if err := r.List(ctx, appPrograms, opts...); err != nil { + return ctrl.Result{Requeue: false}, fmt.Errorf("failed getting BpfApplicationPrograms for full reconcile %s : %v", + req.NamespacedName, err) + } + + if len(appPrograms.Items) == 0 { + r.Logger.Info("BpfApplicationController found no application Programs") + return ctrl.Result{Requeue: false}, nil + } + + var res ctrl.Result + var err error + + for _, a := range appPrograms.Items { + for _, p := range a.Spec.Programs { + switch p.Type { + case bpfmaniov1alpha1.ProgTypeFentry: + fentryProgram := bpfmaniov1alpha1.FentryProgram{ + ObjectMeta: metav1.ObjectMeta{ + Name: a.Name + "fentry", + }, + Spec: bpfmaniov1alpha1.FentryProgramSpec{ + FentryProgramInfo: *p.Fentry, + BpfAppCommon: a.Spec.BpfAppCommon, + }, + } + rec := &FentryProgramReconciler{ + ReconcilerCommon: r.ReconcilerCommon, + currentFentryProgram: &fentryProgram, + } + fentryObjects := []client.Object{&fentryProgram} + // Reconcile FentryProgram. + res, err = r.reconcileCommon(ctx, rec, fentryObjects) + + case bpfmaniov1alpha1.ProgTypeFexit: + fexitProgram := bpfmaniov1alpha1.FexitProgram{ + ObjectMeta: metav1.ObjectMeta{ + Name: a.Name + "fexit", + }, + Spec: bpfmaniov1alpha1.FexitProgramSpec{ + FexitProgramInfo: *p.Fexit, + BpfAppCommon: a.Spec.BpfAppCommon, + }, + } + rec := &FexitProgramReconciler{ + ReconcilerCommon: r.ReconcilerCommon, + currentFexitProgram: &fexitProgram, + } + fexitObjects := []client.Object{&fexitProgram} + // Reconcile FexitProgram. + res, err = r.reconcileCommon(ctx, rec, fexitObjects) + + case bpfmaniov1alpha1.ProgTypeKprobe, + bpfmaniov1alpha1.ProgTypeKretprobe: + kprobeProgram := bpfmaniov1alpha1.KprobeProgram{ + ObjectMeta: metav1.ObjectMeta{ + Name: a.Name + "kprobe", + }, + Spec: bpfmaniov1alpha1.KprobeProgramSpec{ + KprobeProgramInfo: *p.Kprobe, + BpfAppCommon: a.Spec.BpfAppCommon, + }, + } + rec := &KprobeProgramReconciler{ + ReconcilerCommon: r.ReconcilerCommon, + currentKprobeProgram: &kprobeProgram, + } + kprobeObjects := []client.Object{&kprobeProgram} + // Reconcile KprobeProgram or KpretprobeProgram. + res, err = r.reconcileCommon(ctx, rec, kprobeObjects) + + case bpfmaniov1alpha1.ProgTypeUprobe, + bpfmaniov1alpha1.ProgTypeUretprobe: + uprobeProgram := bpfmaniov1alpha1.UprobeProgram{ + ObjectMeta: metav1.ObjectMeta{ + Name: a.Name + "uprobe", + }, + Spec: bpfmaniov1alpha1.UprobeProgramSpec{ + UprobeProgramInfo: *p.Uprobe, + BpfAppCommon: a.Spec.BpfAppCommon, + }, + } + rec := &UprobeProgramReconciler{ + ReconcilerCommon: r.ReconcilerCommon, + currentUprobeProgram: &uprobeProgram, + } + uprobeObjects := []client.Object{&uprobeProgram} + // Reconcile UprobeProgram or UpretprobeProgram. + res, err = r.reconcileCommon(ctx, rec, uprobeObjects) + + case bpfmaniov1alpha1.ProgTypeTracepoint: + tracepointProgram := bpfmaniov1alpha1.TracepointProgram{ + ObjectMeta: metav1.ObjectMeta{ + Name: a.Name + "tracepoint", + }, + Spec: bpfmaniov1alpha1.TracepointProgramSpec{ + TracepointProgramInfo: *p.Tracepoint, + BpfAppCommon: a.Spec.BpfAppCommon, + }, + } + rec := &TracepointProgramReconciler{ + ReconcilerCommon: r.ReconcilerCommon, + currentTracepointProgram: &tracepointProgram, + } + tracepointObjects := []client.Object{&tracepointProgram} + // Reconcile TracepointProgram. + res, err = r.reconcileCommon(ctx, rec, tracepointObjects) + + case bpfmaniov1alpha1.ProgTypeTC, + bpfmaniov1alpha1.ProgTypeTCX: + tcProgram := bpfmaniov1alpha1.TcProgram{ + ObjectMeta: metav1.ObjectMeta{ + Name: a.Name + "tc", + }, + Spec: bpfmaniov1alpha1.TcProgramSpec{ + TcProgramInfo: *p.TC, + BpfAppCommon: a.Spec.BpfAppCommon, + }, + } + rec := &TcProgramReconciler{ + ReconcilerCommon: r.ReconcilerCommon, + currentTcProgram: &tcProgram, + } + tcObjects := []client.Object{&tcProgram} + // Reconcile TcProgram. + res, err = r.reconcileCommon(ctx, rec, tcObjects) + + case bpfmaniov1alpha1.ProgTypeXDP: + xdpProgram := bpfmaniov1alpha1.XdpProgram{ + ObjectMeta: metav1.ObjectMeta{ + Name: a.Name + "xdp", + }, + Spec: bpfmaniov1alpha1.XdpProgramSpec{ + XdpProgramInfo: *p.XDP, + BpfAppCommon: a.Spec.BpfAppCommon, + }, + } + rec := &XdpProgramReconciler{ + ReconcilerCommon: r.ReconcilerCommon, + currentXdpProgram: &xdpProgram, + } + xdpObjects := []client.Object{&xdpProgram} + // Reconcile XdpProgram. + res, err = r.reconcileCommon(ctx, rec, xdpObjects) + + default: + r.Logger.Info("Unsupported Bpf program type", "ProgType", p.Type) + return ctrl.Result{Requeue: false}, nil + } + } + } + + return res, err +} + +// SetupWithManager sets up the controller with the Manager. +// The Bpfman-Agent should reconcile whenever a BpfApplication object is updated, +// load the programs to the node via bpfman, and then create a bpfProgram object +// to reflect per node state information. +func (r *BpfApplicationReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&bpfmaniov1alpha1.BpfApplication{}, builder.WithPredicates(predicate.And(predicate.GenerationChangedPredicate{}, predicate.ResourceVersionChangedPredicate{}))). + Owns(&bpfmaniov1alpha1.BpfProgram{}, + builder.WithPredicates(predicate.And( + internal.BpfProgramNodePredicate(r.NodeName)), + ), + ). + // Only trigger reconciliation if node labels change since that could + // make the BpfApplication no longer select the Node. Additionally only + // care about node events specific to our node + Watches( + &v1.Node{}, + &handler.EnqueueRequestForObject{}, + builder.WithPredicates(predicate.And(predicate.LabelChangedPredicate{}, nodePredicate(r.NodeName))), + ). + Complete(r) +} diff --git a/controllers/bpfman-agent/application-program_test.go b/controllers/bpfman-agent/application-program_test.go new file mode 100644 index 000000000..7f2813408 --- /dev/null +++ b/controllers/bpfman-agent/application-program_test.go @@ -0,0 +1,91 @@ +package bpfmanagent + +import ( + "testing" + + bpfmaniov1alpha1 "github.com/bpfman/bpfman-operator/apis/v1alpha1" + testutils "github.com/bpfman/bpfman-operator/internal/test-utils" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" +) + +func TestBpfApplicationControllerCreate(t *testing.T) { + var ( + name = "fakeAppProgram" + bytecodePath = "/tmp/hello.o" + bpfFentryFunctionName = "fentry_test" + bpfKprobeFunctionName = "kprobe_test" + bpfTracepointFunctionName = "tracepoint-test" + fakeNode = testutils.NewNode("fake-control-plane") + functionFentryName = "do_unlinkat" + functionKprobeName = "try_to_wake_up" + tracepointName = "syscalls/sys_enter_setitimer" + offset = 0 + retprobe = false + ) + + // A AppProgram object with metadata and spec. + App := &bpfmaniov1alpha1.BpfApplication{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: bpfmaniov1alpha1.BpfApplicationSpec{ + BpfAppCommon: bpfmaniov1alpha1.BpfAppCommon{ + NodeSelector: metav1.LabelSelector{}, + }, + Programs: []bpfmaniov1alpha1.BpfApplicationProgram{ + { + Type: bpfmaniov1alpha1.ProgTypeFentry, + Fentry: &bpfmaniov1alpha1.FentryProgramInfo{ + BpfProgramCommon: bpfmaniov1alpha1.BpfProgramCommon{ + BpfFunctionName: bpfFentryFunctionName, + ByteCode: bpfmaniov1alpha1.BytecodeSelector{ + Path: &bytecodePath, + }, + }, + FunctionName: functionFentryName, + }, + }, + { + Type: bpfmaniov1alpha1.ProgTypeKprobe, + Kprobe: &bpfmaniov1alpha1.KprobeProgramInfo{ + BpfProgramCommon: bpfmaniov1alpha1.BpfProgramCommon{ + BpfFunctionName: bpfKprobeFunctionName, + ByteCode: bpfmaniov1alpha1.BytecodeSelector{ + Path: &bytecodePath, + }, + }, + FunctionName: functionKprobeName, + Offset: uint64(offset), + RetProbe: retprobe, + }, + }, + { + Type: bpfmaniov1alpha1.ProgTypeTracepoint, + Tracepoint: &bpfmaniov1alpha1.TracepointProgramInfo{ + BpfProgramCommon: bpfmaniov1alpha1.BpfProgramCommon{ + BpfFunctionName: bpfTracepointFunctionName, + ByteCode: bpfmaniov1alpha1.BytecodeSelector{ + Path: &bytecodePath, + }, + }, + Names: []string{tracepointName}, + }, + }, + }, + }, + } + + // Objects to track in the fake client. + _ = []runtime.Object{fakeNode, App} + + // Register operator types with the runtime scheme. + s := scheme.Scheme + s.AddKnownTypes(bpfmaniov1alpha1.SchemeGroupVersion, App) + s.AddKnownTypes(bpfmaniov1alpha1.SchemeGroupVersion, &bpfmaniov1alpha1.BpfApplicationList{}) + s.AddKnownTypes(bpfmaniov1alpha1.SchemeGroupVersion, &bpfmaniov1alpha1.BpfApplication{}) + s.AddKnownTypes(bpfmaniov1alpha1.SchemeGroupVersion, &bpfmaniov1alpha1.BpfProgramList{}) + +} diff --git a/controllers/bpfman-operator/application-program_test.go b/controllers/bpfman-operator/application-program_test.go new file mode 100644 index 000000000..c4ed5064b --- /dev/null +++ b/controllers/bpfman-operator/application-program_test.go @@ -0,0 +1,221 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bpfmanoperator + +import ( + "context" + "fmt" + "testing" + + bpfmaniov1alpha1 "github.com/bpfman/bpfman-operator/apis/v1alpha1" + internal "github.com/bpfman/bpfman-operator/internal" + testutils "github.com/bpfman/bpfman-operator/internal/test-utils" + + "github.com/stretchr/testify/require" + meta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +// Runs the ApplicationProgramReconcile test. If multiCondition == true, it runs it +// with an error case in which the program object has multiple conditions. +func appProgramReconcile(t *testing.T, multiCondition bool) { + var ( + name = "fakeAppProgram" + bytecodePath = "/tmp/hello.o" + bpfFentryFunctionName = "fentry_test" + bpfKprobeFunctionName = "kprobe_test" + bpfTracepointFunctionName = "tracepoint-test" + fakeNode = testutils.NewNode("fake-control-plane") + functionFentryName = "do_unlinkat" + functionKprobeName = "try_to_wake_up" + tracepointName = "syscalls/sys_enter_setitimer" + offset = 0 + retprobe = false + ctx = context.TODO() + bpfProgName = fmt.Sprintf("%s-%s", name, fakeNode.Name) + ) + // A AppProgram object with metadata and spec. + App := &bpfmaniov1alpha1.BpfApplication{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: bpfmaniov1alpha1.BpfApplicationSpec{ + BpfAppCommon: bpfmaniov1alpha1.BpfAppCommon{ + NodeSelector: metav1.LabelSelector{}, + }, + Programs: []bpfmaniov1alpha1.BpfApplicationProgram{ + { + Type: bpfmaniov1alpha1.ProgTypeFentry, + Fentry: &bpfmaniov1alpha1.FentryProgramInfo{ + BpfProgramCommon: bpfmaniov1alpha1.BpfProgramCommon{ + BpfFunctionName: bpfFentryFunctionName, + ByteCode: bpfmaniov1alpha1.BytecodeSelector{ + Path: &bytecodePath, + }, + }, + FunctionName: functionFentryName, + }, + }, + { + Type: bpfmaniov1alpha1.ProgTypeKprobe, + Kprobe: &bpfmaniov1alpha1.KprobeProgramInfo{ + BpfProgramCommon: bpfmaniov1alpha1.BpfProgramCommon{ + BpfFunctionName: bpfKprobeFunctionName, + ByteCode: bpfmaniov1alpha1.BytecodeSelector{ + Path: &bytecodePath, + }, + }, + FunctionName: functionKprobeName, + Offset: uint64(offset), + RetProbe: retprobe, + }, + }, + { + Type: bpfmaniov1alpha1.ProgTypeTracepoint, + Tracepoint: &bpfmaniov1alpha1.TracepointProgramInfo{ + BpfProgramCommon: bpfmaniov1alpha1.BpfProgramCommon{ + BpfFunctionName: bpfTracepointFunctionName, + ByteCode: bpfmaniov1alpha1.BytecodeSelector{ + Path: &bytecodePath, + }, + }, + Names: []string{tracepointName}, + }, + }, + }, + }, + } + + // The expected accompanying BpfProgram object + expectedBpfProg := &bpfmaniov1alpha1.BpfProgram{ + ObjectMeta: metav1.ObjectMeta{ + Name: bpfProgName, + OwnerReferences: []metav1.OwnerReference{ + { + Name: App.Name, + Controller: &[]bool{true}[0], + }, + }, + Labels: map[string]string{internal.BpfProgramOwnerLabel: App.Name, internal.K8sHostLabel: fakeNode.Name}, + Finalizers: []string{internal.BpfApplicationControllerFinalizer}, + }, + Spec: bpfmaniov1alpha1.BpfProgramSpec{ + Type: "application", + }, + Status: bpfmaniov1alpha1.BpfProgramStatus{ + Conditions: []metav1.Condition{bpfmaniov1alpha1.BpfProgCondLoaded.Condition()}, + }, + } + + // Objects to track in the fake client. + objs := []runtime.Object{fakeNode, App, expectedBpfProg} + + // Register operator types with the runtime scheme. + s := scheme.Scheme + s.AddKnownTypes(bpfmaniov1alpha1.SchemeGroupVersion, App) + s.AddKnownTypes(bpfmaniov1alpha1.SchemeGroupVersion, &bpfmaniov1alpha1.BpfProgram{}) + s.AddKnownTypes(bpfmaniov1alpha1.SchemeGroupVersion, &bpfmaniov1alpha1.BpfProgramList{}) + + // Create a fake client to mock API calls. + cl := fake.NewClientBuilder().WithStatusSubresource(App).WithRuntimeObjects(objs...).Build() + + rc := ReconcilerCommon{ + Client: cl, + Scheme: s, + } + + // Set development Logger so we can see all logs in tests. + logf.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{Development: true}))) + + // Create a ApplicationProgram object with the scheme and fake client. + r := &BpfApplicationReconciler{ReconcilerCommon: rc} + + // Mock request to simulate Reconcile() being called on an event for a + // watched resource . + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: name, + }, + } + + // First reconcile should add the finalizer to the applicationProgram object + res, err := r.Reconcile(ctx, req) + if err != nil { + t.Fatalf("reconcile: (%v)", err) + } + + // Require no requeue + require.False(t, res.Requeue) + + // Check the BpfProgram Object was created successfully + err = cl.Get(ctx, types.NamespacedName{Name: App.Name, Namespace: metav1.NamespaceAll}, App) + require.NoError(t, err) + + // Check the bpfman-operator finalizer was successfully added + require.Contains(t, App.GetFinalizers(), internal.BpfmanOperatorFinalizer) + + // NOTE: THIS IS A TEST FOR AN ERROR PATH. THERE SHOULD NEVER BE MORE THAN + // ONE CONDITION. + if multiCondition { + // Add some random conditions and verify that the condition still gets + // updated correctly. + meta.SetStatusCondition(&App.Status.Conditions, bpfmaniov1alpha1.ProgramDeleteError.Condition("bogus condition #1")) + if err := r.Status().Update(ctx, App); err != nil { + r.Logger.V(1).Info("failed to set App Program object status") + } + meta.SetStatusCondition(&App.Status.Conditions, bpfmaniov1alpha1.ProgramReconcileError.Condition("bogus condition #2")) + if err := r.Status().Update(ctx, App); err != nil { + r.Logger.V(1).Info("failed to set App Program object status") + } + // Make sure we have 2 conditions + require.Equal(t, 2, len(App.Status.Conditions)) + } + + // Second reconcile should check bpfProgram Status and write Success condition to tcProgram Status + res, err = r.Reconcile(ctx, req) + if err != nil { + t.Fatalf("reconcile: (%v)", err) + } + + // Require no requeue + require.False(t, res.Requeue) + + // Check the BpfProgram Object was created successfully + err = cl.Get(ctx, types.NamespacedName{Name: App.Name, Namespace: metav1.NamespaceAll}, App) + require.NoError(t, err) + + // Make sure we only have 1 condition now + require.Equal(t, 1, len(App.Status.Conditions)) + // Make sure it's the right one. + require.Equal(t, App.Status.Conditions[0].Type, string(bpfmaniov1alpha1.ProgramReconcileSuccess)) +} + +func TestAppProgramReconcile(t *testing.T) { + appProgramReconcile(t, false) +} + +func TestAppUpdateStatus(t *testing.T) { + appProgramReconcile(t, true) +} diff --git a/controllers/bpfman-operator/application-programs.go b/controllers/bpfman-operator/application-programs.go index 8e9afa2ab..4357b3878 100644 --- a/controllers/bpfman-operator/application-programs.go +++ b/controllers/bpfman-operator/application-programs.go @@ -1,5 +1,5 @@ /* -Copyright 2023 The bpfman Authors. +Copyright 2024 The bpfman Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -86,62 +86,7 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque } } - return r.reconcileAppPrograms(ctx, appProgram) -} - -func (r *BpfApplicationReconciler) reconcileAppPrograms(ctx context.Context, application *bpfmaniov1alpha1.BpfApplication) (ctrl.Result, error) { - var result ctrl.Result - var err error - - for _, prog := range application.Spec.Programs { - switch prog.Type { - case bpfmaniov1alpha1.ProgTypeXDP: - r.Logger.Info("Reconciling Application XDP Programs") - result, err = reconcileBpfProgram(ctx, r, prog.XDP) - - case bpfmaniov1alpha1.ProgTypeTC: - r.Logger.Info("Reconciling Application TC Programs") - result, err = reconcileBpfProgram(ctx, r, prog.TC) - - case bpfmaniov1alpha1.ProgTypeFentry: - r.Logger.Info("Reconciling Application Fentry/Fexit Programs") - result, err = reconcileBpfProgram(ctx, r, prog.Fentry) - - case bpfmaniov1alpha1.ProgTypeFexit: - r.Logger.Info("Reconciling Application Fexit Programs") - result, err = reconcileBpfProgram(ctx, r, prog.Fexit) - - case bpfmaniov1alpha1.ProgTypeKprobe: - r.Logger.Info("Reconciling Application Kprobe Programs") - result, err = reconcileBpfProgram(ctx, r, prog.Kprobe) - - case bpfmaniov1alpha1.ProgTypeKretprobe: - r.Logger.Info("Reconciling Application Kretprobe Programs") - result, err = reconcileBpfProgram(ctx, r, prog.Kretprobe) - - case bpfmaniov1alpha1.ProgTypeUprobe: - r.Logger.Info("Reconciling Application Uprobe Programs") - result, err = reconcileBpfProgram(ctx, r, prog.Uprobe) - - case bpfmaniov1alpha1.ProgTypeUretprobe: - r.Logger.Info("Reconciling Application Uretprobe Programs") - result, err = reconcileBpfProgram(ctx, r, prog.Uretprobe) - - case bpfmaniov1alpha1.ProgTypeTracepoint: - r.Logger.Info("Reconciling Application Tracepoint Programs") - result, err = reconcileBpfProgram(ctx, r, prog.Tracepoint) - - default: - err := fmt.Errorf("invalid program type: %s", prog.Type) - r.Logger.Error(err, "invalid program type") - return ctrl.Result{}, err - } - if err != nil { - r.Logger.Error(err, "failed reconciling Application Programs") - return ctrl.Result{}, err - } - } - return result, nil + return reconcileBpfProgram(ctx, r, appProgram) } // SetupWithManager sets up the controller with the Manager. From 9d8badcd5aa29897a17e41fb3758a085671e01bb Mon Sep 17 00:00:00 2001 From: Andre Fredette Date: Thu, 6 Jun 2024 15:31:24 -0400 Subject: [PATCH 03/16] Additional changes for BpfApplication object - Ran make generate for latest BpfApplication CRD change - Handle setting owner and object type - If reconcile is complete for one program/object, continue with the next one - Added unit test for fentry TODO: - Add BpfApplication tests for all program types - Add sample BpfApplication yaml and test on kubernetes - See if there's any way to reduce code duplication in application-program.go Signed-off-by: Andre Fredette --- .../crd/bases/bpfman.io_bpfapplications.yaml | 478 ++---------------- .../bpfman-agent/application-program.go | 100 ++-- .../bpfman-agent/application-program_test.go | 185 ++++++- controllers/bpfman-agent/common.go | 43 +- controllers/bpfman-agent/fentry-program.go | 21 +- controllers/bpfman-agent/fexit-program.go | 21 +- controllers/bpfman-agent/kprobe-program.go | 21 +- controllers/bpfman-agent/tc-program.go | 21 +- .../bpfman-agent/tracepoint-program.go | 21 +- controllers/bpfman-agent/uprobe-program.go | 25 +- controllers/bpfman-agent/xdp-program.go | 22 +- .../application-program_test.go | 12 +- internal/constants.go | 1 + 13 files changed, 425 insertions(+), 546 deletions(-) diff --git a/config/crd/bases/bpfman.io_bpfapplications.yaml b/config/crd/bases/bpfman.io_bpfapplications.yaml index 295294ae7..ea865814c 100644 --- a/config/crd/bases/bpfman.io_bpfapplications.yaml +++ b/config/crd/bases/bpfman.io_bpfapplications.yaml @@ -39,6 +39,51 @@ spec: spec: description: BpfApplicationSpec defines the desired state of BpfApplication properties: + bytecode: + description: |- + Bytecode configures where the bpf program's bytecode should be loaded + from. + properties: + image: + description: Image used to specify a bytecode container image. + properties: + imagepullpolicy: + default: IfNotPresent + description: PullPolicy describes a policy for if/when to + pull a bytecode image. Defaults to IfNotPresent. + enum: + - Always + - Never + - IfNotPresent + type: string + imagepullsecret: + description: |- + ImagePullSecret is the name of the secret bpfman should use to get remote image + repository secrets. + properties: + name: + description: Name of the secret which contains the credentials + to access the image repository. + type: string + namespace: + description: Namespace of the secret which contains the + credentials to access the image repository. + type: string + required: + - name + - namespace + type: object + url: + description: Valid container image URL used to reference a + remote bytecode image. + type: string + required: + - url + type: object + path: + description: Path is used to specify a bytecode object via filepath. + type: string + type: object globaldata: additionalProperties: format: byte @@ -116,53 +161,6 @@ spec: BpfFunctionName is the name of the function that is the entry point for the BPF program type: string - bytecode: - description: |- - Bytecode configures where the bpf program's bytecode should be loaded - from. - properties: - image: - description: Image used to specify a bytecode container - image. - properties: - imagepullpolicy: - default: IfNotPresent - description: PullPolicy describes a policy for if/when - to pull a bytecode image. Defaults to IfNotPresent. - enum: - - Always - - Never - - IfNotPresent - type: string - imagepullsecret: - description: |- - ImagePullSecret is the name of the secret bpfman should use to get remote image - repository secrets. - properties: - name: - description: Name of the secret which contains - the credentials to access the image repository. - type: string - namespace: - description: Namespace of the secret which contains - the credentials to access the image repository. - type: string - required: - - name - - namespace - type: object - url: - description: Valid container image URL used to reference - a remote bytecode image. - type: string - required: - - url - type: object - path: - description: Path is used to specify a bytecode object - via filepath. - type: string - type: object func_name: description: Function to attach the fentry to. type: string @@ -218,7 +216,6 @@ spec: x-kubernetes-map-type: atomic required: - bpffunctionname - - bytecode - func_name type: object fexit: @@ -230,53 +227,6 @@ spec: BpfFunctionName is the name of the function that is the entry point for the BPF program type: string - bytecode: - description: |- - Bytecode configures where the bpf program's bytecode should be loaded - from. - properties: - image: - description: Image used to specify a bytecode container - image. - properties: - imagepullpolicy: - default: IfNotPresent - description: PullPolicy describes a policy for if/when - to pull a bytecode image. Defaults to IfNotPresent. - enum: - - Always - - Never - - IfNotPresent - type: string - imagepullsecret: - description: |- - ImagePullSecret is the name of the secret bpfman should use to get remote image - repository secrets. - properties: - name: - description: Name of the secret which contains - the credentials to access the image repository. - type: string - namespace: - description: Namespace of the secret which contains - the credentials to access the image repository. - type: string - required: - - name - - namespace - type: object - url: - description: Valid container image URL used to reference - a remote bytecode image. - type: string - required: - - url - type: object - path: - description: Path is used to specify a bytecode object - via filepath. - type: string - type: object func_name: description: Function to attach the fexit to. type: string @@ -332,7 +282,6 @@ spec: x-kubernetes-map-type: atomic required: - bpffunctionname - - bytecode - func_name type: object kprobe: @@ -344,53 +293,6 @@ spec: BpfFunctionName is the name of the function that is the entry point for the BPF program type: string - bytecode: - description: |- - Bytecode configures where the bpf program's bytecode should be loaded - from. - properties: - image: - description: Image used to specify a bytecode container - image. - properties: - imagepullpolicy: - default: IfNotPresent - description: PullPolicy describes a policy for if/when - to pull a bytecode image. Defaults to IfNotPresent. - enum: - - Always - - Never - - IfNotPresent - type: string - imagepullsecret: - description: |- - ImagePullSecret is the name of the secret bpfman should use to get remote image - repository secrets. - properties: - name: - description: Name of the secret which contains - the credentials to access the image repository. - type: string - namespace: - description: Namespace of the secret which contains - the credentials to access the image repository. - type: string - required: - - name - - namespace - type: object - url: - description: Valid container image URL used to reference - a remote bytecode image. - type: string - required: - - url - type: object - path: - description: Path is used to specify a bytecode object - via filepath. - type: string - type: object func_name: description: Functions to attach the kprobe to. type: string @@ -458,7 +360,6 @@ spec: type: boolean required: - bpffunctionname - - bytecode - func_name type: object kretprobe: @@ -470,53 +371,6 @@ spec: BpfFunctionName is the name of the function that is the entry point for the BPF program type: string - bytecode: - description: |- - Bytecode configures where the bpf program's bytecode should be loaded - from. - properties: - image: - description: Image used to specify a bytecode container - image. - properties: - imagepullpolicy: - default: IfNotPresent - description: PullPolicy describes a policy for if/when - to pull a bytecode image. Defaults to IfNotPresent. - enum: - - Always - - Never - - IfNotPresent - type: string - imagepullsecret: - description: |- - ImagePullSecret is the name of the secret bpfman should use to get remote image - repository secrets. - properties: - name: - description: Name of the secret which contains - the credentials to access the image repository. - type: string - namespace: - description: Namespace of the secret which contains - the credentials to access the image repository. - type: string - required: - - name - - namespace - type: object - url: - description: Valid container image URL used to reference - a remote bytecode image. - type: string - required: - - url - type: object - path: - description: Path is used to specify a bytecode object - via filepath. - type: string - type: object func_name: description: Functions to attach the kprobe to. type: string @@ -584,7 +438,6 @@ spec: type: boolean required: - bpffunctionname - - bytecode - func_name type: object tc: @@ -596,53 +449,6 @@ spec: BpfFunctionName is the name of the function that is the entry point for the BPF program type: string - bytecode: - description: |- - Bytecode configures where the bpf program's bytecode should be loaded - from. - properties: - image: - description: Image used to specify a bytecode container - image. - properties: - imagepullpolicy: - default: IfNotPresent - description: PullPolicy describes a policy for if/when - to pull a bytecode image. Defaults to IfNotPresent. - enum: - - Always - - Never - - IfNotPresent - type: string - imagepullsecret: - description: |- - ImagePullSecret is the name of the secret bpfman should use to get remote image - repository secrets. - properties: - name: - description: Name of the secret which contains - the credentials to access the image repository. - type: string - namespace: - description: Namespace of the secret which contains - the credentials to access the image repository. - type: string - required: - - name - - namespace - type: object - url: - description: Valid container image URL used to reference - a remote bytecode image. - type: string - required: - - url - type: object - path: - description: Path is used to specify a bytecode object - via filepath. - type: string - type: object direction: description: |- Direction specifies the direction of traffic the tc program should @@ -753,7 +559,6 @@ spec: type: array required: - bpffunctionname - - bytecode - direction - interfaceselector - priority @@ -767,53 +572,6 @@ spec: BpfFunctionName is the name of the function that is the entry point for the BPF program type: string - bytecode: - description: |- - Bytecode configures where the bpf program's bytecode should be loaded - from. - properties: - image: - description: Image used to specify a bytecode container - image. - properties: - imagepullpolicy: - default: IfNotPresent - description: PullPolicy describes a policy for if/when - to pull a bytecode image. Defaults to IfNotPresent. - enum: - - Always - - Never - - IfNotPresent - type: string - imagepullsecret: - description: |- - ImagePullSecret is the name of the secret bpfman should use to get remote image - repository secrets. - properties: - name: - description: Name of the secret which contains - the credentials to access the image repository. - type: string - namespace: - description: Namespace of the secret which contains - the credentials to access the image repository. - type: string - required: - - name - - namespace - type: object - url: - description: Valid container image URL used to reference - a remote bytecode image. - type: string - required: - - url - type: object - path: - description: Path is used to specify a bytecode object - via filepath. - type: string - type: object mapownerselector: description: |- MapOwnerSelector is used to select the loaded eBPF program this eBPF program @@ -873,7 +631,6 @@ spec: type: array required: - bpffunctionname - - bytecode - names type: object type: @@ -899,53 +656,6 @@ spec: BpfFunctionName is the name of the function that is the entry point for the BPF program type: string - bytecode: - description: |- - Bytecode configures where the bpf program's bytecode should be loaded - from. - properties: - image: - description: Image used to specify a bytecode container - image. - properties: - imagepullpolicy: - default: IfNotPresent - description: PullPolicy describes a policy for if/when - to pull a bytecode image. Defaults to IfNotPresent. - enum: - - Always - - Never - - IfNotPresent - type: string - imagepullsecret: - description: |- - ImagePullSecret is the name of the secret bpfman should use to get remote image - repository secrets. - properties: - name: - description: Name of the secret which contains - the credentials to access the image repository. - type: string - namespace: - description: Namespace of the secret which contains - the credentials to access the image repository. - type: string - required: - - name - - namespace - type: object - url: - description: Valid container image URL used to reference - a remote bytecode image. - type: string - required: - - url - type: object - path: - description: Path is used to specify a bytecode object - via filepath. - type: string - type: object containers: description: |- Containers identifes the set of containers in which to attach the uprobe. @@ -1093,7 +803,6 @@ spec: type: string required: - bpffunctionname - - bytecode - target type: object uretprobe: @@ -1105,53 +814,6 @@ spec: BpfFunctionName is the name of the function that is the entry point for the BPF program type: string - bytecode: - description: |- - Bytecode configures where the bpf program's bytecode should be loaded - from. - properties: - image: - description: Image used to specify a bytecode container - image. - properties: - imagepullpolicy: - default: IfNotPresent - description: PullPolicy describes a policy for if/when - to pull a bytecode image. Defaults to IfNotPresent. - enum: - - Always - - Never - - IfNotPresent - type: string - imagepullsecret: - description: |- - ImagePullSecret is the name of the secret bpfman should use to get remote image - repository secrets. - properties: - name: - description: Name of the secret which contains - the credentials to access the image repository. - type: string - namespace: - description: Namespace of the secret which contains - the credentials to access the image repository. - type: string - required: - - name - - namespace - type: object - url: - description: Valid container image URL used to reference - a remote bytecode image. - type: string - required: - - url - type: object - path: - description: Path is used to specify a bytecode object - via filepath. - type: string - type: object containers: description: |- Containers identifes the set of containers in which to attach the uprobe. @@ -1299,7 +961,6 @@ spec: type: string required: - bpffunctionname - - bytecode - target type: object xdp: @@ -1311,53 +972,6 @@ spec: BpfFunctionName is the name of the function that is the entry point for the BPF program type: string - bytecode: - description: |- - Bytecode configures where the bpf program's bytecode should be loaded - from. - properties: - image: - description: Image used to specify a bytecode container - image. - properties: - imagepullpolicy: - default: IfNotPresent - description: PullPolicy describes a policy for if/when - to pull a bytecode image. Defaults to IfNotPresent. - enum: - - Always - - Never - - IfNotPresent - type: string - imagepullsecret: - description: |- - ImagePullSecret is the name of the secret bpfman should use to get remote image - repository secrets. - properties: - name: - description: Name of the secret which contains - the credentials to access the image repository. - type: string - namespace: - description: Namespace of the secret which contains - the credentials to access the image repository. - type: string - required: - - name - - namespace - type: object - url: - description: Valid container image URL used to reference - a remote bytecode image. - type: string - required: - - url - type: object - path: - description: Path is used to specify a bytecode object - via filepath. - type: string - type: object interfaceselector: description: Selector to determine the network interface (or interfaces) @@ -1452,7 +1066,6 @@ spec: type: array required: - bpffunctionname - - bytecode - interfaceselector - priority type: object @@ -1460,6 +1073,7 @@ spec: minItems: 1 type: array required: + - bytecode - nodeselector type: object status: diff --git a/controllers/bpfman-agent/application-program.go b/controllers/bpfman-agent/application-program.go index 64d131afb..92c35aaa5 100644 --- a/controllers/bpfman-agent/application-program.go +++ b/controllers/bpfman-agent/application-program.go @@ -24,31 +24,40 @@ type BpfApplicationReconciler struct { ourNode *v1.Node } -func (r *BpfApplicationReconciler) getFinalizer() string { - return internal.BpfApplicationControllerFinalizer -} +// TODO: As implemented, the BpfApplicationReconciler doesn't need to implement +// the bpfmanReconciler interface. We should think about what's needed and what +// isn't. -func (r *BpfApplicationReconciler) getName() string { - return r.currentApp.Name -} +// func (r *BpfApplicationReconciler) getFinalizer() string { +// return internal.BpfApplicationControllerFinalizer +// } -func (r *BpfApplicationReconciler) getNode() *v1.Node { - return r.ourNode -} +// func (r *BpfApplicationReconciler) getName() string { +// return r.currentApp.Name +// } -func (r *BpfApplicationReconciler) getNodeSelector() *metav1.LabelSelector { - return &r.currentApp.Spec.NodeSelector +func (r *BpfApplicationReconciler) getRecType() string { + return internal.ApplicationString } -func (r *BpfApplicationReconciler) getBpfGlobalData() map[string][]byte { - return r.currentApp.Spec.GlobalData -} +// func (r *BpfApplicationReconciler) getNode() *v1.Node { +// return r.ourNode +// } + +// func (r *BpfApplicationReconciler) getNodeSelector() *metav1.LabelSelector { +// return &r.currentApp.Spec.NodeSelector +// } + +// func (r *BpfApplicationReconciler) getBpfGlobalData() map[string][]byte { +// return r.currentApp.Spec.GlobalData +// } func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { // Initialize node and current program r.currentApp = &bpfmaniov1alpha1.BpfApplication{} r.ourNode = &v1.Node{} r.Logger = ctrl.Log.WithName("application") + r.appOwner = &bpfmaniov1alpha1.BpfApplication{} ctxLogger := log.FromContext(ctx) ctxLogger.Info("Reconcile Application: Enter", "ReconcileKey", req) @@ -75,6 +84,7 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque var res ctrl.Result var err error + var complete bool for _, a := range appPrograms.Items { for _, p := range a.Spec.Programs { @@ -82,7 +92,7 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque case bpfmaniov1alpha1.ProgTypeFentry: fentryProgram := bpfmaniov1alpha1.FentryProgram{ ObjectMeta: metav1.ObjectMeta{ - Name: a.Name + "fentry", + Name: a.Name + "-fentry", }, Spec: bpfmaniov1alpha1.FentryProgramSpec{ FentryProgramInfo: *p.Fentry, @@ -92,15 +102,17 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque rec := &FentryProgramReconciler{ ReconcilerCommon: r.ReconcilerCommon, currentFentryProgram: &fentryProgram, + ourNode: r.ourNode, } + rec.appOwner = &a fentryObjects := []client.Object{&fentryProgram} // Reconcile FentryProgram. - res, err = r.reconcileCommon(ctx, rec, fentryObjects) + complete, res, err = r.reconcileCommon(ctx, rec, fentryObjects) case bpfmaniov1alpha1.ProgTypeFexit: fexitProgram := bpfmaniov1alpha1.FexitProgram{ ObjectMeta: metav1.ObjectMeta{ - Name: a.Name + "fexit", + Name: a.Name + "-fexit", }, Spec: bpfmaniov1alpha1.FexitProgramSpec{ FexitProgramInfo: *p.Fexit, @@ -110,16 +122,18 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque rec := &FexitProgramReconciler{ ReconcilerCommon: r.ReconcilerCommon, currentFexitProgram: &fexitProgram, + ourNode: r.ourNode, } + rec.appOwner = &a fexitObjects := []client.Object{&fexitProgram} // Reconcile FexitProgram. - res, err = r.reconcileCommon(ctx, rec, fexitObjects) + complete, res, err = r.reconcileCommon(ctx, rec, fexitObjects) case bpfmaniov1alpha1.ProgTypeKprobe, bpfmaniov1alpha1.ProgTypeKretprobe: kprobeProgram := bpfmaniov1alpha1.KprobeProgram{ ObjectMeta: metav1.ObjectMeta{ - Name: a.Name + "kprobe", + Name: a.Name + "-kprobe", }, Spec: bpfmaniov1alpha1.KprobeProgramSpec{ KprobeProgramInfo: *p.Kprobe, @@ -129,16 +143,18 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque rec := &KprobeProgramReconciler{ ReconcilerCommon: r.ReconcilerCommon, currentKprobeProgram: &kprobeProgram, + ourNode: r.ourNode, } + rec.appOwner = &a kprobeObjects := []client.Object{&kprobeProgram} // Reconcile KprobeProgram or KpretprobeProgram. - res, err = r.reconcileCommon(ctx, rec, kprobeObjects) + complete, res, err = r.reconcileCommon(ctx, rec, kprobeObjects) case bpfmaniov1alpha1.ProgTypeUprobe, bpfmaniov1alpha1.ProgTypeUretprobe: uprobeProgram := bpfmaniov1alpha1.UprobeProgram{ ObjectMeta: metav1.ObjectMeta{ - Name: a.Name + "uprobe", + Name: a.Name + "-uprobe", }, Spec: bpfmaniov1alpha1.UprobeProgramSpec{ UprobeProgramInfo: *p.Uprobe, @@ -148,15 +164,17 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque rec := &UprobeProgramReconciler{ ReconcilerCommon: r.ReconcilerCommon, currentUprobeProgram: &uprobeProgram, + ourNode: r.ourNode, } + rec.appOwner = &a uprobeObjects := []client.Object{&uprobeProgram} // Reconcile UprobeProgram or UpretprobeProgram. - res, err = r.reconcileCommon(ctx, rec, uprobeObjects) + complete, res, err = r.reconcileCommon(ctx, rec, uprobeObjects) case bpfmaniov1alpha1.ProgTypeTracepoint: tracepointProgram := bpfmaniov1alpha1.TracepointProgram{ ObjectMeta: metav1.ObjectMeta{ - Name: a.Name + "tracepoint", + Name: a.Name + "-tracepoint", }, Spec: bpfmaniov1alpha1.TracepointProgramSpec{ TracepointProgramInfo: *p.Tracepoint, @@ -166,16 +184,18 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque rec := &TracepointProgramReconciler{ ReconcilerCommon: r.ReconcilerCommon, currentTracepointProgram: &tracepointProgram, + ourNode: r.ourNode, } + rec.appOwner = &a tracepointObjects := []client.Object{&tracepointProgram} // Reconcile TracepointProgram. - res, err = r.reconcileCommon(ctx, rec, tracepointObjects) + complete, res, err = r.reconcileCommon(ctx, rec, tracepointObjects) case bpfmaniov1alpha1.ProgTypeTC, bpfmaniov1alpha1.ProgTypeTCX: tcProgram := bpfmaniov1alpha1.TcProgram{ ObjectMeta: metav1.ObjectMeta{ - Name: a.Name + "tc", + Name: a.Name + "-tc", }, Spec: bpfmaniov1alpha1.TcProgramSpec{ TcProgramInfo: *p.TC, @@ -185,15 +205,17 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque rec := &TcProgramReconciler{ ReconcilerCommon: r.ReconcilerCommon, currentTcProgram: &tcProgram, + ourNode: r.ourNode, } + rec.appOwner = &a tcObjects := []client.Object{&tcProgram} // Reconcile TcProgram. - res, err = r.reconcileCommon(ctx, rec, tcObjects) + complete, res, err = r.reconcileCommon(ctx, rec, tcObjects) case bpfmaniov1alpha1.ProgTypeXDP: xdpProgram := bpfmaniov1alpha1.XdpProgram{ ObjectMeta: metav1.ObjectMeta{ - Name: a.Name + "xdp", + Name: a.Name + "-xdp", }, Spec: bpfmaniov1alpha1.XdpProgramSpec{ XdpProgramInfo: *p.XDP, @@ -203,16 +225,33 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque rec := &XdpProgramReconciler{ ReconcilerCommon: r.ReconcilerCommon, currentXdpProgram: &xdpProgram, + ourNode: r.ourNode, } + rec.appOwner = &a xdpObjects := []client.Object{&xdpProgram} // Reconcile XdpProgram. - res, err = r.reconcileCommon(ctx, rec, xdpObjects) + complete, res, err = r.reconcileCommon(ctx, rec, xdpObjects) default: - r.Logger.Info("Unsupported Bpf program type", "ProgType", p.Type) - return ctrl.Result{Requeue: false}, nil + r.Logger.Error(fmt.Errorf("Unsupported Bpf program type"), "Unsupported Bpf program type", "ProgType", p.Type) + // Skip this program and continue to the next one + continue + } + + if complete { + // We've completed reconciling this program, continue to the next one + continue + } else { + return res, err } } + + if complete { + // We've completed reconciling all programs for this application, continue to the next one + continue + } else { + return res, err + } } return res, err @@ -227,6 +266,7 @@ func (r *BpfApplicationReconciler) SetupWithManager(mgr ctrl.Manager) error { For(&bpfmaniov1alpha1.BpfApplication{}, builder.WithPredicates(predicate.And(predicate.GenerationChangedPredicate{}, predicate.ResourceVersionChangedPredicate{}))). Owns(&bpfmaniov1alpha1.BpfProgram{}, builder.WithPredicates(predicate.And( + internal.BpfProgramTypePredicate(internal.ApplicationString), internal.BpfProgramNodePredicate(r.NodeName)), ), ). diff --git a/controllers/bpfman-agent/application-program_test.go b/controllers/bpfman-agent/application-program_test.go index 7f2813408..5d26ebba9 100644 --- a/controllers/bpfman-agent/application-program_test.go +++ b/controllers/bpfman-agent/application-program_test.go @@ -1,29 +1,52 @@ package bpfmanagent import ( + "context" + "fmt" "testing" bpfmaniov1alpha1 "github.com/bpfman/bpfman-operator/apis/v1alpha1" + bpfmanagentinternal "github.com/bpfman/bpfman-operator/controllers/bpfman-agent/internal" + agenttestutils "github.com/bpfman/bpfman-operator/controllers/bpfman-agent/internal/test-utils" + "github.com/bpfman/bpfman-operator/internal" testutils "github.com/bpfman/bpfman-operator/internal/test-utils" + gobpfman "github.com/bpfman/bpfman/clients/gobpfman/v1" + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/testing/protocmp" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/reconcile" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" ) func TestBpfApplicationControllerCreate(t *testing.T) { var ( - name = "fakeAppProgram" - bytecodePath = "/tmp/hello.o" - bpfFentryFunctionName = "fentry_test" - bpfKprobeFunctionName = "kprobe_test" - bpfTracepointFunctionName = "tracepoint-test" - fakeNode = testutils.NewNode("fake-control-plane") - functionFentryName = "do_unlinkat" - functionKprobeName = "try_to_wake_up" + // global config + name = "fakeAppProgram" + namespace = "bpfman" + bytecodePath = "/tmp/hello.o" + fakeNode = testutils.NewNode("fake-control-plane") + ctx = context.TODO() + // fentry program config + fentryBpfFunctionName = "fentry_test" + fentryFunctionName = "do_unlinkat" + fentryBpfProgName = fmt.Sprintf("%s-%s-%s-%s", name, "fentry", fakeNode.Name, "do-unlinkat") + fentryBpfProg = &bpfmaniov1alpha1.BpfProgram{} + fentryFakeUID = "ef71d42c-aa21-48e8-a697-82391d801a81" + // kprobe program config + kprobeBpfFunctionName = "kprobe_test" + kprobeFunctionName = "try_to_wake_up" + kprobeOffset = 0 + kprobeRetprobe = false + // tracepoint program config + tracepointBpfFunctionName = "tracepoint_test" tracepointName = "syscalls/sys_enter_setitimer" - offset = 0 - retprobe = false ) // A AppProgram object with metadata and spec. @@ -34,42 +57,36 @@ func TestBpfApplicationControllerCreate(t *testing.T) { Spec: bpfmaniov1alpha1.BpfApplicationSpec{ BpfAppCommon: bpfmaniov1alpha1.BpfAppCommon{ NodeSelector: metav1.LabelSelector{}, + ByteCode: bpfmaniov1alpha1.BytecodeSelector{ + Path: &bytecodePath, + }, }, Programs: []bpfmaniov1alpha1.BpfApplicationProgram{ { Type: bpfmaniov1alpha1.ProgTypeFentry, Fentry: &bpfmaniov1alpha1.FentryProgramInfo{ BpfProgramCommon: bpfmaniov1alpha1.BpfProgramCommon{ - BpfFunctionName: bpfFentryFunctionName, - ByteCode: bpfmaniov1alpha1.BytecodeSelector{ - Path: &bytecodePath, - }, + BpfFunctionName: fentryBpfFunctionName, }, - FunctionName: functionFentryName, + FunctionName: fentryFunctionName, }, }, { Type: bpfmaniov1alpha1.ProgTypeKprobe, Kprobe: &bpfmaniov1alpha1.KprobeProgramInfo{ BpfProgramCommon: bpfmaniov1alpha1.BpfProgramCommon{ - BpfFunctionName: bpfKprobeFunctionName, - ByteCode: bpfmaniov1alpha1.BytecodeSelector{ - Path: &bytecodePath, - }, + BpfFunctionName: kprobeBpfFunctionName, }, - FunctionName: functionKprobeName, - Offset: uint64(offset), - RetProbe: retprobe, + FunctionName: kprobeFunctionName, + Offset: uint64(kprobeOffset), + RetProbe: kprobeRetprobe, }, }, { Type: bpfmaniov1alpha1.ProgTypeTracepoint, Tracepoint: &bpfmaniov1alpha1.TracepointProgramInfo{ BpfProgramCommon: bpfmaniov1alpha1.BpfProgramCommon{ - BpfFunctionName: bpfTracepointFunctionName, - ByteCode: bpfmaniov1alpha1.BytecodeSelector{ - Path: &bytecodePath, - }, + BpfFunctionName: tracepointBpfFunctionName, }, Names: []string{tracepointName}, }, @@ -79,7 +96,7 @@ func TestBpfApplicationControllerCreate(t *testing.T) { } // Objects to track in the fake client. - _ = []runtime.Object{fakeNode, App} + objs := []runtime.Object{fakeNode, App} // Register operator types with the runtime scheme. s := scheme.Scheme @@ -87,5 +104,119 @@ func TestBpfApplicationControllerCreate(t *testing.T) { s.AddKnownTypes(bpfmaniov1alpha1.SchemeGroupVersion, &bpfmaniov1alpha1.BpfApplicationList{}) s.AddKnownTypes(bpfmaniov1alpha1.SchemeGroupVersion, &bpfmaniov1alpha1.BpfApplication{}) s.AddKnownTypes(bpfmaniov1alpha1.SchemeGroupVersion, &bpfmaniov1alpha1.BpfProgramList{}) + s.AddKnownTypes(bpfmaniov1alpha1.SchemeGroupVersion, &bpfmaniov1alpha1.BpfProgram{}) + s.AddKnownTypes(bpfmaniov1alpha1.SchemeGroupVersion, &bpfmaniov1alpha1.FentryProgramList{}) + + // Create a fake client to mock API calls. + cl := fake.NewClientBuilder().WithStatusSubresource(App).WithStatusSubresource(&bpfmaniov1alpha1.BpfProgram{}).WithRuntimeObjects(objs...).Build() + + cli := agenttestutils.NewBpfmanClientFake() + + rc := ReconcilerCommon{ + Client: cl, + Scheme: s, + BpfmanClient: cli, + NodeName: fakeNode.Name, + appOwner: App, + } + + // Set development Logger so we can see all logs in tests. + logf.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{Development: true}))) + + // Create a ReconcileMemcached object with the scheme and fake client. + r := &BpfApplicationReconciler{ReconcilerCommon: rc, ourNode: fakeNode} + + // Mock request to simulate Reconcile() being called on an event for a + // watched resource . + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: name, + Namespace: namespace, + }, + } + + // First reconcile should create the bpf program object + res, err := r.Reconcile(ctx, req) + if err != nil { + t.Fatalf("reconcile: (%v)", err) + } + + // Check the BpfProgram Object was created successfully + err = cl.Get(ctx, types.NamespacedName{Name: fentryBpfProgName, Namespace: metav1.NamespaceAll}, fentryBpfProg) + require.NoError(t, err) + + require.NotEmpty(t, fentryBpfProg) + // Finalizer is written + require.Equal(t, internal.FentryProgramControllerFinalizer, fentryBpfProg.Finalizers[0]) + // owningConfig Label was correctly set + require.Equal(t, name, fentryBpfProg.Labels[internal.BpfProgramOwnerLabel]) + // node Label was correctly set + require.Equal(t, fakeNode.Name, fentryBpfProg.Labels[internal.K8sHostLabel]) + // fentry function Annotation was correctly set + require.Equal(t, fentryFunctionName, fentryBpfProg.Annotations[internal.FentryProgramFunction]) + // Type is set + require.Equal(t, r.getRecType(), fentryBpfProg.Spec.Type) + // Require no requeue + require.False(t, res.Requeue) + + // Update UID of bpfProgram with Fake UID since the fake API server won't + fentryBpfProg.UID = types.UID(fentryFakeUID) + err = cl.Update(ctx, fentryBpfProg) + require.NoError(t, err) + + // Second reconcile should create the bpfman Load Request and update the + // BpfProgram object's maps field and id annotation. + res, err = r.Reconcile(ctx, req) + if err != nil { + t.Fatalf("reconcile: (%v)", err) + } + + // Require no requeue + require.False(t, res.Requeue) + expectedLoadReq := &gobpfman.LoadRequest{ + Bytecode: &gobpfman.BytecodeLocation{ + Location: &gobpfman.BytecodeLocation_File{File: bytecodePath}, + }, + Name: fentryBpfFunctionName, + ProgramType: *internal.Tracing.Uint32(), + Metadata: map[string]string{internal.UuidMetadataKey: string(fentryBpfProg.UID), internal.ProgramNameKey: name}, + MapOwnerId: nil, + Attach: &gobpfman.AttachInfo{ + Info: &gobpfman.AttachInfo_FentryAttachInfo{ + FentryAttachInfo: &gobpfman.FentryAttachInfo{ + FnName: fentryFunctionName, + }, + }, + }, + } + + // Check that the bpfProgram's programs was correctly updated + err = cl.Get(ctx, types.NamespacedName{Name: fentryBpfProgName, Namespace: metav1.NamespaceAll}, fentryBpfProg) + require.NoError(t, err) + + // prog ID should already have been set + id, err := bpfmanagentinternal.GetID(fentryBpfProg) + require.NoError(t, err) + + // Check the bpfLoadRequest was correctly Built + if !cmp.Equal(expectedLoadReq, cli.LoadRequests[int(*id)], protocmp.Transform()) { + cmp.Diff(expectedLoadReq, cli.LoadRequests[int(*id)], protocmp.Transform()) + t.Logf("Diff %v", cmp.Diff(expectedLoadReq, cli.LoadRequests[int(*id)], protocmp.Transform())) + t.Fatal("Built bpfman LoadRequest does not match expected") + } + + // Third reconcile should set the status to loaded + res, err = r.Reconcile(ctx, req) + if err != nil { + t.Fatalf("reconcile: (%v)", err) + } + + // Require no requeue + require.False(t, res.Requeue) + + // Check that the bpfProgram's status was correctly updated + err = cl.Get(ctx, types.NamespacedName{Name: fentryBpfProgName, Namespace: metav1.NamespaceAll}, fentryBpfProg) + require.NoError(t, err) + require.Equal(t, string(bpfmaniov1alpha1.BpfProgCondLoaded), fentryBpfProg.Status.Conditions[0].Type) } diff --git a/controllers/bpfman-agent/common.go b/controllers/bpfman-agent/common.go index 6485dd417..540a56e21 100644 --- a/controllers/bpfman-agent/common.go +++ b/controllers/bpfman-agent/common.go @@ -71,6 +71,7 @@ type ReconcilerCommon struct { Logger logr.Logger NodeName string progId *uint32 + appOwner metav1.Object // Set if the owner is an application } // bpfmanReconciler defines a generic bpfProgram K8s object reconciler which can @@ -88,6 +89,9 @@ type bpfmanReconciler interface { // getFinalizer returns the string used for the finalizer to prevent the // BpfProgram object from deletion until cleanup can be performed getFinalizer() string + // getOwner returns the owner of the BpfProgram object. This is either the + // *Program or the BpfApplicationProgram that created it. + getOwner() metav1.Object // getRecType returns the type of the reconciler. This is often the string // representation of the ProgramType, but in cases where there are multiple // reconcilers for a single ProgramType, it may be different (e.g., uprobe, @@ -119,13 +123,16 @@ type bpfmanReconciler interface { } // reconcileCommon is the common reconciler loop called by each bpfman -// reconciler. It reconciles each program in the list. reconcileCommon should -// not return error because it will trigger an infinite reconcile loop. -// Instead, it should report the error to user and retry if specified. For some -// errors the controller may decide not to retry. Note: This only results in -// calls to bpfman if we need to change something +// reconciler. It reconciles each program in the list. The boolean return +// value is set to true if we've made it through all the programs in the list +// without anything being updated and a reque has not been requested. Otherwise, +// it's set to false. reconcileCommon should not return error because it will +// trigger an infinite reconcile loop. Instead, it should report the error to +// user and retry if specified. For some errors the controller may decide not to +// retry. Note: This only results in calls to bpfman if we need to change +// something func (r *ReconcilerCommon) reconcileCommon(ctx context.Context, rec bpfmanReconciler, - programs []client.Object) (ctrl.Result, error) { + programs []client.Object) (bool, ctrl.Result, error) { r.Logger.V(1).Info("Start reconcileCommon()") @@ -133,7 +140,7 @@ func (r *ReconcilerCommon) reconcileCommon(ctx context.Context, rec bpfmanReconc loadedBpfPrograms, err := bpfmanagentinternal.ListBpfmanPrograms(ctx, r.BpfmanClient, rec.getProgType()) if err != nil { r.Logger.Error(err, "failed to list loaded bpfman programs") - return ctrl.Result{Requeue: true, RequeueAfter: retryDurationAgent}, nil + return false, ctrl.Result{Requeue: true, RequeueAfter: retryDurationAgent}, nil } requeue := false // initialize requeue to false @@ -144,7 +151,7 @@ func (r *ReconcilerCommon) reconcileCommon(ctx context.Context, rec bpfmanReconc err := rec.setCurrentProgram(program) if err != nil { r.Logger.Error(err, "Failed to set current program") - return ctrl.Result{Requeue: true, RequeueAfter: retryDurationAgent}, nil + return false, ctrl.Result{Requeue: true, RequeueAfter: retryDurationAgent}, nil } result, err := r.reconcileProgram(ctx, rec, program, loadedBpfPrograms) @@ -157,7 +164,7 @@ func (r *ReconcilerCommon) reconcileCommon(ctx context.Context, rec bpfmanReconc // continue with next program case internal.Updated: // return - return ctrl.Result{Requeue: false}, nil + return false, ctrl.Result{Requeue: false}, nil case internal.Requeue: // remember to do a requeue when we're done and continue with next program requeue = true @@ -166,11 +173,11 @@ func (r *ReconcilerCommon) reconcileCommon(ctx context.Context, rec bpfmanReconc if requeue { // A requeue has been requested - return ctrl.Result{RequeueAfter: retryDurationAgent}, nil + return false, ctrl.Result{RequeueAfter: retryDurationAgent}, nil } else { // We've made it through all the programs in the list without anything being // updated and a reque has not been requested. - return ctrl.Result{Requeue: false}, nil + return true, ctrl.Result{Requeue: false}, nil } } @@ -491,13 +498,13 @@ func (r *ReconcilerCommon) updateStatus(ctx context.Context, bpfProgram *bpfmani } func (r *ReconcilerCommon) getExistingBpfPrograms(ctx context.Context, - program metav1.Object) (map[string]bpfmaniov1alpha1.BpfProgram, error) { + owner string) (map[string]bpfmaniov1alpha1.BpfProgram, error) { bpfProgramList := &bpfmaniov1alpha1.BpfProgramList{} // Only list bpfPrograms for this *Program and the controller's node opts := []client.ListOption{ - client.MatchingLabels{internal.BpfProgramOwnerLabel: program.GetName(), internal.K8sHostLabel: r.NodeName}, + client.MatchingLabels{internal.BpfProgramOwnerLabel: owner, internal.K8sHostLabel: r.NodeName}, } err := r.List(ctx, bpfProgramList, opts...) @@ -521,6 +528,9 @@ func (r *ReconcilerCommon) createBpfProgram( owner metav1.Object, ownerType string, annotations map[string]string) (*bpfmaniov1alpha1.BpfProgram, error) { + + r.Logger.V(1).Info("createBpfProgram()", "Name", bpfProgramName, "Owner", owner.GetName(), "OwnerType", ownerType) + bpfProg := &bpfmaniov1alpha1.BpfProgram{ ObjectMeta: metav1.ObjectMeta{ Name: bpfProgramName, @@ -624,7 +634,6 @@ func (r *ReconcilerCommon) handleProgDelete( func (r *ReconcilerCommon) handleProgCreateOrUpdate( ctx context.Context, rec bpfmanReconciler, - program client.Object, existingBpfPrograms map[string]bpfmaniov1alpha1.BpfProgram, expectedBpfPrograms *bpfmaniov1alpha1.BpfProgramList, loadedBpfPrograms map[string]*gobpfman.ListResponse_ListResult, @@ -646,7 +655,7 @@ func (r *ReconcilerCommon) handleProgCreateOrUpdate( } else { // Create a new bpfProgram Object for this program. opts := client.CreateOptions{} - r.Logger.Info("Creating bpfProgram", "Name", expectedBpfProgram.Name, "Owner", program.GetName()) + r.Logger.Info("Creating bpfProgram", "Name", expectedBpfProgram.Name, "Owner", rec.getOwner().GetName()) if err := r.Create(ctx, &expectedBpfProgram, &opts); err != nil { return internal.Requeue, fmt.Errorf("failed to create bpfProgram object: %v", err) } @@ -742,7 +751,7 @@ func (r *ReconcilerCommon) reconcileProgram(ctx context.Context, // Query the K8s API to get a list of existing bpfPrograms for this *Program // on this node. - existingBpfPrograms, err := r.getExistingBpfPrograms(ctx, program) + existingBpfPrograms, err := r.getExistingBpfPrograms(ctx, rec.getOwner().GetName()) if err != nil { return internal.Requeue, fmt.Errorf("failed to get existing bpfPrograms: %v", err) } @@ -772,7 +781,7 @@ func (r *ReconcilerCommon) reconcileProgram(ctx context.Context, if err != nil { return internal.Requeue, fmt.Errorf("failed to get expected bpfPrograms: %v", err) } - return r.handleProgCreateOrUpdate(ctx, rec, program, existingBpfPrograms, expectedBpfPrograms, loadedBpfPrograms, + return r.handleProgCreateOrUpdate(ctx, rec, existingBpfPrograms, expectedBpfPrograms, loadedBpfPrograms, isNodeSelected, isBeingDeleted, mapOwnerStatus) } diff --git a/controllers/bpfman-agent/fentry-program.go b/controllers/bpfman-agent/fentry-program.go index 872f6b77d..eb72d3cc3 100644 --- a/controllers/bpfman-agent/fentry-program.go +++ b/controllers/bpfman-agent/fentry-program.go @@ -50,8 +50,20 @@ func (r *FentryProgramReconciler) getFinalizer() string { return internal.FentryProgramControllerFinalizer } +func (r *FentryProgramReconciler) getOwner() metav1.Object { + if r.appOwner == nil { + return r.currentFentryProgram + } else { + return r.appOwner + } +} + func (r *FentryProgramReconciler) getRecType() string { - return internal.FentryString + if r.appOwner == nil { + return internal.FentryString + } else { + return internal.ApplicationString + } } func (r *FentryProgramReconciler) getProgType() internal.ProgramType { @@ -122,7 +134,7 @@ func (r *FentryProgramReconciler) getExpectedBpfPrograms(ctx context.Context) (* annotations := map[string]string{internal.FentryProgramFunction: r.currentFentryProgram.Spec.FunctionName} - prog, err := r.createBpfProgram(bpfProgramName, r.getFinalizer(), r.currentFentryProgram, r.getRecType(), annotations) + prog, err := r.createBpfProgram(bpfProgramName, r.getFinalizer(), r.getOwner(), r.getRecType(), annotations) if err != nil { return nil, fmt.Errorf("failed to create BpfProgram %s: %v", bpfProgramName, err) } @@ -168,7 +180,8 @@ func (r *FentryProgramReconciler) Reconcile(ctx context.Context, req ctrl.Reques } // Reconcile each FentryProgram. - return r.reconcileCommon(ctx, r, fentryObjects) + _, result, err := r.reconcileCommon(ctx, r, fentryObjects) + return result, err } func (r *FentryProgramReconciler) getLoadRequest(bpfProgram *bpfmaniov1alpha1.BpfProgram, mapOwnerId *uint32) (*gobpfman.LoadRequest, error) { @@ -188,7 +201,7 @@ func (r *FentryProgramReconciler) getLoadRequest(bpfProgram *bpfmaniov1alpha1.Bp }, }, }, - Metadata: map[string]string{internal.UuidMetadataKey: string(bpfProgram.UID), internal.ProgramNameKey: r.currentFentryProgram.Name}, + Metadata: map[string]string{internal.UuidMetadataKey: string(bpfProgram.UID), internal.ProgramNameKey: r.getOwner().GetName()}, GlobalData: r.currentFentryProgram.Spec.GlobalData, MapOwnerId: mapOwnerId, } diff --git a/controllers/bpfman-agent/fexit-program.go b/controllers/bpfman-agent/fexit-program.go index ca45313cd..cde85e989 100644 --- a/controllers/bpfman-agent/fexit-program.go +++ b/controllers/bpfman-agent/fexit-program.go @@ -50,8 +50,20 @@ func (r *FexitProgramReconciler) getFinalizer() string { return internal.FexitProgramControllerFinalizer } +func (r *FexitProgramReconciler) getOwner() metav1.Object { + if r.appOwner == nil { + return r.currentFexitProgram + } else { + return r.appOwner + } +} + func (r *FexitProgramReconciler) getRecType() string { - return internal.FexitString + if r.appOwner == nil { + return internal.FexitString + } else { + return internal.ApplicationString + } } func (r *FexitProgramReconciler) getProgType() internal.ProgramType { @@ -122,7 +134,7 @@ func (r *FexitProgramReconciler) getExpectedBpfPrograms(ctx context.Context) (*b annotations := map[string]string{internal.FexitProgramFunction: r.currentFexitProgram.Spec.FunctionName} - prog, err := r.createBpfProgram(bpfProgramName, r.getFinalizer(), r.currentFexitProgram, r.getRecType(), annotations) + prog, err := r.createBpfProgram(bpfProgramName, r.getFinalizer(), r.getOwner(), r.getRecType(), annotations) if err != nil { return nil, fmt.Errorf("failed to create BpfProgram %s: %v", bpfProgramName, err) } @@ -168,7 +180,8 @@ func (r *FexitProgramReconciler) Reconcile(ctx context.Context, req ctrl.Request } // Reconcile each FexitProgram. - return r.reconcileCommon(ctx, r, fexitObjects) + _, result, err := r.reconcileCommon(ctx, r, fexitObjects) + return result, err } func (r *FexitProgramReconciler) getLoadRequest(bpfProgram *bpfmaniov1alpha1.BpfProgram, mapOwnerId *uint32) (*gobpfman.LoadRequest, error) { @@ -188,7 +201,7 @@ func (r *FexitProgramReconciler) getLoadRequest(bpfProgram *bpfmaniov1alpha1.Bpf }, }, }, - Metadata: map[string]string{internal.UuidMetadataKey: string(bpfProgram.UID), internal.ProgramNameKey: r.currentFexitProgram.Name}, + Metadata: map[string]string{internal.UuidMetadataKey: string(bpfProgram.UID), internal.ProgramNameKey: r.getOwner().GetName()}, GlobalData: r.currentFexitProgram.Spec.GlobalData, MapOwnerId: mapOwnerId, } diff --git a/controllers/bpfman-agent/kprobe-program.go b/controllers/bpfman-agent/kprobe-program.go index 01766e103..5d0cd8123 100644 --- a/controllers/bpfman-agent/kprobe-program.go +++ b/controllers/bpfman-agent/kprobe-program.go @@ -50,8 +50,20 @@ func (r *KprobeProgramReconciler) getFinalizer() string { return internal.KprobeProgramControllerFinalizer } +func (r *KprobeProgramReconciler) getOwner() metav1.Object { + if r.appOwner == nil { + return r.currentKprobeProgram + } else { + return r.appOwner + } +} + func (r *KprobeProgramReconciler) getRecType() string { - return internal.Kprobe.String() + if r.appOwner == nil { + return internal.Kprobe.String() + } else { + return internal.ApplicationString + } } func (r *KprobeProgramReconciler) getProgType() internal.ProgramType { @@ -122,7 +134,7 @@ func (r *KprobeProgramReconciler) getExpectedBpfPrograms(ctx context.Context) (* annotations := map[string]string{internal.KprobeProgramFunction: r.currentKprobeProgram.Spec.FunctionName} - prog, err := r.createBpfProgram(bpfProgramName, r.getFinalizer(), r.currentKprobeProgram, r.getRecType(), annotations) + prog, err := r.createBpfProgram(bpfProgramName, r.getFinalizer(), r.getOwner(), r.getRecType(), annotations) if err != nil { return nil, fmt.Errorf("failed to create BpfProgram %s: %v", bpfProgramName, err) } @@ -168,7 +180,8 @@ func (r *KprobeProgramReconciler) Reconcile(ctx context.Context, req ctrl.Reques } // Reconcile each KprobeProgram. - return r.reconcileCommon(ctx, r, kprobeObjects) + _, result, err := r.reconcileCommon(ctx, r, kprobeObjects) + return result, err } func (r *KprobeProgramReconciler) getLoadRequest(bpfProgram *bpfmaniov1alpha1.BpfProgram, mapOwnerId *uint32) (*gobpfman.LoadRequest, error) { @@ -194,7 +207,7 @@ func (r *KprobeProgramReconciler) getLoadRequest(bpfProgram *bpfmaniov1alpha1.Bp }, }, }, - Metadata: map[string]string{internal.UuidMetadataKey: string(bpfProgram.UID), internal.ProgramNameKey: r.currentKprobeProgram.Name}, + Metadata: map[string]string{internal.UuidMetadataKey: string(bpfProgram.UID), internal.ProgramNameKey: r.getOwner().GetName()}, GlobalData: r.currentKprobeProgram.Spec.GlobalData, MapOwnerId: mapOwnerId, } diff --git a/controllers/bpfman-agent/tc-program.go b/controllers/bpfman-agent/tc-program.go index 935b04f26..f4fc403dc 100644 --- a/controllers/bpfman-agent/tc-program.go +++ b/controllers/bpfman-agent/tc-program.go @@ -51,8 +51,20 @@ func (r *TcProgramReconciler) getFinalizer() string { return internal.TcProgramControllerFinalizer } +func (r *TcProgramReconciler) getOwner() metav1.Object { + if r.appOwner == nil { + return r.currentTcProgram + } else { + return r.appOwner + } +} + func (r *TcProgramReconciler) getRecType() string { - return internal.Tc.String() + if r.appOwner == nil { + return internal.Tc.String() + } else { + return internal.ApplicationString + } } func (r *TcProgramReconciler) getProgType() internal.ProgramType { @@ -164,7 +176,7 @@ func (r *TcProgramReconciler) getExpectedBpfPrograms(ctx context.Context) (*bpfm bpfProgramName := fmt.Sprintf("%s-%s-%s", r.currentTcProgram.Name, r.NodeName, iface) annotations := map[string]string{internal.TcProgramInterface: iface} - prog, err := r.createBpfProgram(bpfProgramName, r.getFinalizer(), r.currentTcProgram, r.getRecType(), annotations) + prog, err := r.createBpfProgram(bpfProgramName, r.getFinalizer(), r.getOwner(), r.getRecType(), annotations) if err != nil { return nil, fmt.Errorf("failed to create BpfProgram %s: %v", bpfProgramName, err) } @@ -211,7 +223,8 @@ func (r *TcProgramReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( } // Reconcile each TcProgram. - return r.reconcileCommon(ctx, r, tcObjects) + _, result, err := r.reconcileCommon(ctx, r, tcObjects) + return result, err } func (r *TcProgramReconciler) getLoadRequest(bpfProgram *bpfmaniov1alpha1.BpfProgram, mapOwnerId *uint32) (*gobpfman.LoadRequest, error) { @@ -234,7 +247,7 @@ func (r *TcProgramReconciler) getLoadRequest(bpfProgram *bpfmaniov1alpha1.BpfPro }, }, }, - Metadata: map[string]string{internal.UuidMetadataKey: string(bpfProgram.UID), internal.ProgramNameKey: r.currentTcProgram.Name}, + Metadata: map[string]string{internal.UuidMetadataKey: string(bpfProgram.UID), internal.ProgramNameKey: r.getOwner().GetName()}, GlobalData: r.currentTcProgram.Spec.GlobalData, MapOwnerId: mapOwnerId, } diff --git a/controllers/bpfman-agent/tracepoint-program.go b/controllers/bpfman-agent/tracepoint-program.go index fdc00bf53..6e410e387 100644 --- a/controllers/bpfman-agent/tracepoint-program.go +++ b/controllers/bpfman-agent/tracepoint-program.go @@ -50,8 +50,20 @@ func (r *TracepointProgramReconciler) getFinalizer() string { return internal.TracepointProgramControllerFinalizer } +func (r *TracepointProgramReconciler) getOwner() metav1.Object { + if r.appOwner == nil { + return r.currentTracepointProgram + } else { + return r.appOwner + } +} + func (r *TracepointProgramReconciler) getRecType() string { - return internal.Tracepoint.String() + if r.appOwner == nil { + return internal.Tracepoint.String() + } else { + return internal.ApplicationString + } } func (r *TracepointProgramReconciler) getProgType() internal.ProgramType { @@ -123,7 +135,7 @@ func (r *TracepointProgramReconciler) getExpectedBpfPrograms(ctx context.Context annotations := map[string]string{internal.TracepointProgramTracepoint: tracepoint} - prog, err := r.createBpfProgram(bpfProgramName, r.getFinalizer(), r.currentTracepointProgram, r.getRecType(), annotations) + prog, err := r.createBpfProgram(bpfProgramName, r.getFinalizer(), r.getOwner(), r.getRecType(), annotations) if err != nil { return nil, fmt.Errorf("failed to create BpfProgram %s: %v", bpfProgramName, err) } @@ -170,7 +182,8 @@ func (r *TracepointProgramReconciler) Reconcile(ctx context.Context, req ctrl.Re } // Reconcile each TcProgram. - return r.reconcileCommon(ctx, r, tracepointObjects) + _, result, err := r.reconcileCommon(ctx, r, tracepointObjects) + return result, err } func (r *TracepointProgramReconciler) getLoadRequest(bpfProgram *bpfmaniov1alpha1.BpfProgram, mapOwnerId *uint32) (*gobpfman.LoadRequest, error) { @@ -190,7 +203,7 @@ func (r *TracepointProgramReconciler) getLoadRequest(bpfProgram *bpfmaniov1alpha }, }, }, - Metadata: map[string]string{internal.UuidMetadataKey: string(bpfProgram.UID), internal.ProgramNameKey: r.currentTracepointProgram.Name}, + Metadata: map[string]string{internal.UuidMetadataKey: string(bpfProgram.UID), internal.ProgramNameKey: r.getOwner().GetName()}, GlobalData: r.currentTracepointProgram.Spec.GlobalData, MapOwnerId: mapOwnerId, } diff --git a/controllers/bpfman-agent/uprobe-program.go b/controllers/bpfman-agent/uprobe-program.go index c1092ae01..2fa428b7c 100644 --- a/controllers/bpfman-agent/uprobe-program.go +++ b/controllers/bpfman-agent/uprobe-program.go @@ -51,8 +51,20 @@ func (r *UprobeProgramReconciler) getFinalizer() string { return internal.UprobeProgramControllerFinalizer } +func (r *UprobeProgramReconciler) getOwner() metav1.Object { + if r.appOwner == nil { + return r.currentUprobeProgram + } else { + return r.appOwner + } +} + func (r *UprobeProgramReconciler) getRecType() string { - return internal.UprobeString + if r.appOwner == nil { + return internal.UprobeString + } else { + return internal.ApplicationString + } } func (r *UprobeProgramReconciler) getProgType() internal.ProgramType { @@ -171,7 +183,7 @@ func (r *UprobeProgramReconciler) getExpectedBpfPrograms(ctx context.Context) (* bpfProgramName := fmt.Sprintf("%s-%s", bpfProgramNameBase, "no-containers-on-node") - prog, err := r.createBpfProgram(bpfProgramName, r.getFinalizer(), r.currentUprobeProgram, r.getRecType(), annotations) + prog, err := r.createBpfProgram(bpfProgramName, r.getFinalizer(), r.getOwner(), r.getRecType(), annotations) if err != nil { return nil, fmt.Errorf("failed to create BpfProgram %s: %v", bpfProgramNameBase, err) } @@ -188,7 +200,7 @@ func (r *UprobeProgramReconciler) getExpectedBpfPrograms(ctx context.Context) (* bpfProgramName := fmt.Sprintf("%s-%s-%s", bpfProgramNameBase, container.podName, container.containerName) - prog, err := r.createBpfProgram(bpfProgramName, r.getFinalizer(), r.currentUprobeProgram, r.getRecType(), annotations) + prog, err := r.createBpfProgram(bpfProgramName, r.getFinalizer(), r.getOwner(), r.getRecType(), annotations) if err != nil { return nil, fmt.Errorf("failed to create BpfProgram %s: %v", bpfProgramName, err) } @@ -199,7 +211,7 @@ func (r *UprobeProgramReconciler) getExpectedBpfPrograms(ctx context.Context) (* } else { annotations := map[string]string{internal.UprobeProgramTarget: r.currentUprobeProgram.Spec.Target} - prog, err := r.createBpfProgram(bpfProgramNameBase, r.getFinalizer(), r.currentUprobeProgram, r.getRecType(), annotations) + prog, err := r.createBpfProgram(bpfProgramNameBase, r.getFinalizer(), r.getOwner(), r.getRecType(), annotations) if err != nil { return nil, fmt.Errorf("failed to create BpfProgram %s: %v", bpfProgramNameBase, err) } @@ -246,7 +258,8 @@ func (r *UprobeProgramReconciler) Reconcile(ctx context.Context, req ctrl.Reques } // Reconcile each TcProgram. - return r.reconcileCommon(ctx, r, uprobeObjects) + _, result, err := r.reconcileCommon(ctx, r, uprobeObjects) + return result, err } func (r *UprobeProgramReconciler) getLoadRequest(bpfProgram *bpfmaniov1alpha1.BpfProgram, mapOwnerId *uint32) (*gobpfman.LoadRequest, error) { @@ -292,7 +305,7 @@ func (r *UprobeProgramReconciler) getLoadRequest(bpfProgram *bpfmaniov1alpha1.Bp UprobeAttachInfo: uprobeAttachInfo, }, }, - Metadata: map[string]string{internal.UuidMetadataKey: string(bpfProgram.UID), internal.ProgramNameKey: r.currentUprobeProgram.Name}, + Metadata: map[string]string{internal.UuidMetadataKey: string(bpfProgram.UID), internal.ProgramNameKey: r.getOwner().GetName()}, GlobalData: r.currentUprobeProgram.Spec.GlobalData, MapOwnerId: mapOwnerId, } diff --git a/controllers/bpfman-agent/xdp-program.go b/controllers/bpfman-agent/xdp-program.go index ff185f668..30d9a1526 100644 --- a/controllers/bpfman-agent/xdp-program.go +++ b/controllers/bpfman-agent/xdp-program.go @@ -50,10 +50,21 @@ func (r *XdpProgramReconciler) getFinalizer() string { return internal.XdpProgramControllerFinalizer } -func (r *XdpProgramReconciler) getRecType() string { - return internal.Xdp.String() +func (r *XdpProgramReconciler) getOwner() metav1.Object { + if r.appOwner == nil { + return r.currentXdpProgram + } else { + return r.appOwner + } } +func (r *XdpProgramReconciler) getRecType() string { + if r.appOwner == nil { + return internal.Xdp.String() + } else { + return internal.ApplicationString + } +} func (r *XdpProgramReconciler) getProgType() internal.ProgramType { return internal.Xdp } @@ -150,7 +161,7 @@ func (r *XdpProgramReconciler) getExpectedBpfPrograms(ctx context.Context) (*bpf bpfProgramName := fmt.Sprintf("%s-%s-%s", r.currentXdpProgram.Name, r.NodeName, iface) annotations := map[string]string{internal.XdpProgramInterface: iface} - prog, err := r.createBpfProgram(bpfProgramName, r.getFinalizer(), r.currentXdpProgram, r.getRecType(), annotations) + prog, err := r.createBpfProgram(bpfProgramName, r.getFinalizer(), r.getOwner(), r.getRecType(), annotations) if err != nil { return nil, fmt.Errorf("failed to create BpfProgram %s: %v", bpfProgramName, err) } @@ -196,7 +207,8 @@ func (r *XdpProgramReconciler) Reconcile(ctx context.Context, req ctrl.Request) } // Reconcile each TcProgram. - return r.reconcileCommon(ctx, r, xdpObjects) + _, result, err := r.reconcileCommon(ctx, r, xdpObjects) + return result, err } func (r *XdpProgramReconciler) getLoadRequest(bpfProgram *bpfmaniov1alpha1.BpfProgram, mapOwnerId *uint32) (*gobpfman.LoadRequest, error) { @@ -218,7 +230,7 @@ func (r *XdpProgramReconciler) getLoadRequest(bpfProgram *bpfmaniov1alpha1.BpfPr }, }, }, - Metadata: map[string]string{internal.UuidMetadataKey: string(bpfProgram.UID), internal.ProgramNameKey: r.currentXdpProgram.Name}, + Metadata: map[string]string{internal.UuidMetadataKey: string(bpfProgram.UID), internal.ProgramNameKey: r.getOwner().GetName()}, GlobalData: r.currentXdpProgram.Spec.GlobalData, MapOwnerId: mapOwnerId, } diff --git a/controllers/bpfman-operator/application-program_test.go b/controllers/bpfman-operator/application-program_test.go index c4ed5064b..257b977a4 100644 --- a/controllers/bpfman-operator/application-program_test.go +++ b/controllers/bpfman-operator/application-program_test.go @@ -64,6 +64,9 @@ func appProgramReconcile(t *testing.T, multiCondition bool) { Spec: bpfmaniov1alpha1.BpfApplicationSpec{ BpfAppCommon: bpfmaniov1alpha1.BpfAppCommon{ NodeSelector: metav1.LabelSelector{}, + ByteCode: bpfmaniov1alpha1.BytecodeSelector{ + Path: &bytecodePath, + }, }, Programs: []bpfmaniov1alpha1.BpfApplicationProgram{ { @@ -71,9 +74,6 @@ func appProgramReconcile(t *testing.T, multiCondition bool) { Fentry: &bpfmaniov1alpha1.FentryProgramInfo{ BpfProgramCommon: bpfmaniov1alpha1.BpfProgramCommon{ BpfFunctionName: bpfFentryFunctionName, - ByteCode: bpfmaniov1alpha1.BytecodeSelector{ - Path: &bytecodePath, - }, }, FunctionName: functionFentryName, }, @@ -83,9 +83,6 @@ func appProgramReconcile(t *testing.T, multiCondition bool) { Kprobe: &bpfmaniov1alpha1.KprobeProgramInfo{ BpfProgramCommon: bpfmaniov1alpha1.BpfProgramCommon{ BpfFunctionName: bpfKprobeFunctionName, - ByteCode: bpfmaniov1alpha1.BytecodeSelector{ - Path: &bytecodePath, - }, }, FunctionName: functionKprobeName, Offset: uint64(offset), @@ -97,9 +94,6 @@ func appProgramReconcile(t *testing.T, multiCondition bool) { Tracepoint: &bpfmaniov1alpha1.TracepointProgramInfo{ BpfProgramCommon: bpfmaniov1alpha1.BpfProgramCommon{ BpfFunctionName: bpfTracepointFunctionName, - ByteCode: bpfmaniov1alpha1.BytecodeSelector{ - Path: &bytecodePath, - }, }, Names: []string{tracepointName}, }, diff --git a/internal/constants.go b/internal/constants.go index 473108805..a98317748 100644 --- a/internal/constants.go +++ b/internal/constants.go @@ -230,6 +230,7 @@ func (p ProgramType) String() string { const UprobeString = "uprobe" const FentryString = "fentry" const FexitString = "fexit" +const ApplicationString = "application" type ReconcileResult uint8 From 1cbf1e30b8fd62db5df1a1905e74efee17d2e92f Mon Sep 17 00:00:00 2001 From: Andre Fredette Date: Fri, 7 Jun 2024 15:51:45 -0400 Subject: [PATCH 04/16] Set up k8s permissions for bpfapplication Signed-off-by: Andre Fredette --- PROJECT | 130 +++++++++--------- apis/v1alpha1/bpfapplication_types.go | 2 +- .../kustomization.yaml | 2 +- config/rbac/bpfman-agent/role.yaml | 14 ++ .../bpfman-agent/application-program.go | 2 + controllers/bpfman-agent/common.go | 1 + .../bpfman-operator/application-programs.go | 9 +- 7 files changed, 89 insertions(+), 71 deletions(-) diff --git a/PROJECT b/PROJECT index 7ee64d607..f4d69833a 100644 --- a/PROJECT +++ b/PROJECT @@ -4,75 +4,75 @@ # More info: https://book.kubebuilder.io/reference/project-config.html domain: bpfman.io layout: -- go.kubebuilder.io/v3 + - go.kubebuilder.io/v3 plugins: manifests.sdk.operatorframework.io/v2: {} scorecard.sdk.operatorframework.io/v2: {} projectName: bpfman-operator repo: github.com/bpfman resources: -- api: - crdVersion: v1 - namespaced: true - controller: true - domain: bpfman.io - kind: BpfProgram - path: github.com/bpfman/bpfman-operator/apis/v1alpha1 - version: v1alpha1 -- api: - crdVersion: v1 - controller: true - domain: bpfman.io - kind: XdpProgram - path: github.com/bpfman/bpfman-operator/apis/v1alpha1 - version: v1alpha1 -- api: - crdVersion: v1 - controller: true - domain: bpfman.io - kind: TcProgram - path: github.com/bpfman/bpfman-operator/apis/v1alpha1 - version: v1alpha1 -- api: - crdVersion: v1 - controller: true - domain: bpfman.io - kind: TracePointProgram - path: github.com/bpfman/bpfman-operator/apis/v1alpha1 - version: v1alpha1 -- api: - crdVersion: v1 - controller: true - domain: bpfman.io - kind: KprobeProgram - path: github.com/bpfman/bpfman-operator/apis/v1alpha1 - version: v1alpha1 -- api: - crdVersion: v1 - controller: true - domain: bpfman.io - kind: UprobeProgram - path: github.com/bpfman/bpfman-operator/apis/v1alpha1 - version: v1alpha1 -- api: - crdVersion: v1 - controller: true - domain: bpfman.io - kind: FentryProgram - path: github.com/bpfman/bpfman-operator/apis/v1alpha1 - version: v1alpha1 -- api: - crdVersion: v1 - controller: true - domain: bpfman.io - kind: FexitProgram - path: github.com/bpfman/bpfman-operator/apis/v1alpha1 - version: v1alpha1 -- api: - crdVersion: v1 - controller: true - domain: bpfman.io - kind: BpfApplication - path: github.com/bpfman/api/v1alpha1 - version: v1alpha1 + - api: + crdVersion: v1 + namespaced: true + controller: true + domain: bpfman.io + kind: BpfProgram + path: github.com/bpfman/bpfman-operator/apis/v1alpha1 + version: v1alpha1 + - api: + crdVersion: v1 + controller: true + domain: bpfman.io + kind: XdpProgram + path: github.com/bpfman/bpfman-operator/apis/v1alpha1 + version: v1alpha1 + - api: + crdVersion: v1 + controller: true + domain: bpfman.io + kind: TcProgram + path: github.com/bpfman/bpfman-operator/apis/v1alpha1 + version: v1alpha1 + - api: + crdVersion: v1 + controller: true + domain: bpfman.io + kind: TracePointProgram + path: github.com/bpfman/bpfman-operator/apis/v1alpha1 + version: v1alpha1 + - api: + crdVersion: v1 + controller: true + domain: bpfman.io + kind: KprobeProgram + path: github.com/bpfman/bpfman-operator/apis/v1alpha1 + version: v1alpha1 + - api: + crdVersion: v1 + controller: true + domain: bpfman.io + kind: UprobeProgram + path: github.com/bpfman/bpfman-operator/apis/v1alpha1 + version: v1alpha1 + - api: + crdVersion: v1 + controller: true + domain: bpfman.io + kind: FentryProgram + path: github.com/bpfman/bpfman-operator/apis/v1alpha1 + version: v1alpha1 + - api: + crdVersion: v1 + controller: true + domain: bpfman.io + kind: FexitProgram + path: github.com/bpfman/bpfman-operator/apis/v1alpha1 + version: v1alpha1 + - api: + crdVersion: v1 + controller: true + domain: bpfman.io + kind: BpfApplication + path: github.com/bpfman/bpfman-operator/apis/v1alpha1 + version: v1alpha1 version: "3" diff --git a/apis/v1alpha1/bpfapplication_types.go b/apis/v1alpha1/bpfapplication_types.go index 40d11c889..6bc1fbcc5 100644 --- a/apis/v1alpha1/bpfapplication_types.go +++ b/apis/v1alpha1/bpfapplication_types.go @@ -142,7 +142,7 @@ type BpfApplication struct { } // +kubebuilder:object:root=true -// BpfApplicationList contains a list of BpfApplication +// BpfApplicationList contains a list of BpfApplications type BpfApplicationList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` diff --git a/config/bpfman-operator-deployment/kustomization.yaml b/config/bpfman-operator-deployment/kustomization.yaml index d0eb3dd30..816ab6601 100644 --- a/config/bpfman-operator-deployment/kustomization.yaml +++ b/config/bpfman-operator-deployment/kustomization.yaml @@ -5,4 +5,4 @@ kind: Kustomization images: - name: quay.io/bpfman/bpfman-operator newName: quay.io/bpfman/bpfman-operator - newTag: latest + newTag: latest-amd64 diff --git a/config/rbac/bpfman-agent/role.yaml b/config/rbac/bpfman-agent/role.yaml index 23d4e9d68..4cc0e9fe9 100644 --- a/config/rbac/bpfman-agent/role.yaml +++ b/config/rbac/bpfman-agent/role.yaml @@ -4,6 +4,20 @@ kind: ClusterRole metadata: name: agent-role rules: +- apiGroups: + - bpfman.io + resources: + - bpfapplications + verbs: + - get + - list + - watch +- apiGroups: + - bpfman.io + resources: + - bpfapplications/finalizers + verbs: + - update - apiGroups: - bpfman.io resources: diff --git a/controllers/bpfman-agent/application-program.go b/controllers/bpfman-agent/application-program.go index 92c35aaa5..c806dfe28 100644 --- a/controllers/bpfman-agent/application-program.go +++ b/controllers/bpfman-agent/application-program.go @@ -18,6 +18,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" ) +//+kubebuilder:rbac:groups=bpfman.io,resources=bpfapplications,verbs=get;list;watch + type BpfApplicationReconciler struct { ReconcilerCommon currentApp *bpfmaniov1alpha1.BpfApplication diff --git a/controllers/bpfman-agent/common.go b/controllers/bpfman-agent/common.go index 540a56e21..124d1eaed 100644 --- a/controllers/bpfman-agent/common.go +++ b/controllers/bpfman-agent/common.go @@ -54,6 +54,7 @@ import ( //+kubebuilder:rbac:groups=bpfman.io,resources=uprobeprograms/finalizers,verbs=update //+kubebuilder:rbac:groups=bpfman.io,resources=fentryprograms/finalizers,verbs=update //+kubebuilder:rbac:groups=bpfman.io,resources=fexityprograms/finalizers,verbs=update +//+kubebuilder:rbac:groups=bpfman.io,resources=bpfapplications/finalizers,verbs=update //+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch //+kubebuilder:rbac:groups=core,resources=nodes,verbs=get;list;watch //+kubebuilder:rbac:groups=core,resources=secrets,verbs=get diff --git a/controllers/bpfman-operator/application-programs.go b/controllers/bpfman-operator/application-programs.go index 4357b3878..64998a9b7 100644 --- a/controllers/bpfman-operator/application-programs.go +++ b/controllers/bpfman-operator/application-programs.go @@ -19,6 +19,7 @@ package bpfmanoperator import ( "context" "fmt" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -31,6 +32,10 @@ import ( internal "github.com/bpfman/bpfman-operator/internal" ) +//+kubebuilder:rbac:groups=bpfman.io,resources=bpfapplications,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=bpfman.io,resources=bpfapplications/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=bpfman.io,resources=bpfapplications/finalizers,verbs=update + // BpfApplicationReconciler reconciles a BpfApplication object type BpfApplicationReconciler struct { ReconcilerCommon @@ -44,10 +49,6 @@ func (r *BpfApplicationReconciler) getFinalizer() string { return internal.BpfApplicationControllerFinalizer } -//+kubebuilder:rbac:groups=bpfman.io,resources=bpfapplications,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=bpfman.io,resources=bpfapplications/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=bpfman.io,resources=bpfapplications/finalizers,verbs=update - func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { r.Logger = log.FromContext(ctx) From d3d497dbfb94f9cc38039fcf6489d94b8e5662ec Mon Sep 17 00:00:00 2001 From: Andre Fredette Date: Fri, 7 Jun 2024 16:16:34 -0400 Subject: [PATCH 05/16] Fix bundle manifest check error Signed-off-by: Andre Fredette --- ...c.authorization.k8s.io_v1_clusterrole.yaml | 14 + .../manifests/bpfman.io_bpfapplications.yaml | 1168 +++++++++++++++++ .../kustomization.yaml | 2 +- config/samples/kustomization.yaml | 2 +- 4 files changed, 1184 insertions(+), 2 deletions(-) create mode 100644 bundle/manifests/bpfman.io_bpfapplications.yaml diff --git a/bundle/manifests/bpfman-agent-role_rbac.authorization.k8s.io_v1_clusterrole.yaml b/bundle/manifests/bpfman-agent-role_rbac.authorization.k8s.io_v1_clusterrole.yaml index d3046c6b3..a051e457d 100644 --- a/bundle/manifests/bpfman-agent-role_rbac.authorization.k8s.io_v1_clusterrole.yaml +++ b/bundle/manifests/bpfman-agent-role_rbac.authorization.k8s.io_v1_clusterrole.yaml @@ -4,6 +4,20 @@ metadata: creationTimestamp: null name: bpfman-agent-role rules: +- apiGroups: + - bpfman.io + resources: + - bpfapplications + verbs: + - get + - list + - watch +- apiGroups: + - bpfman.io + resources: + - bpfapplications/finalizers + verbs: + - update - apiGroups: - bpfman.io resources: diff --git a/bundle/manifests/bpfman.io_bpfapplications.yaml b/bundle/manifests/bpfman.io_bpfapplications.yaml new file mode 100644 index 000000000..6053bd26f --- /dev/null +++ b/bundle/manifests/bpfman.io_bpfapplications.yaml @@ -0,0 +1,1168 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + creationTimestamp: null + name: bpfapplications.bpfman.io +spec: + group: bpfman.io + names: + kind: BpfApplication + listKind: BpfApplicationList + plural: bpfapplications + singular: bpfapplication + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: BpfApplication is the Schema for the bpfapplications API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: BpfApplicationSpec defines the desired state of BpfApplication + properties: + bytecode: + description: |- + Bytecode configures where the bpf program's bytecode should be loaded + from. + properties: + image: + description: Image used to specify a bytecode container image. + properties: + imagepullpolicy: + default: IfNotPresent + description: PullPolicy describes a policy for if/when to + pull a bytecode image. Defaults to IfNotPresent. + enum: + - Always + - Never + - IfNotPresent + type: string + imagepullsecret: + description: |- + ImagePullSecret is the name of the secret bpfman should use to get remote image + repository secrets. + properties: + name: + description: Name of the secret which contains the credentials + to access the image repository. + type: string + namespace: + description: Namespace of the secret which contains the + credentials to access the image repository. + type: string + required: + - name + - namespace + type: object + url: + description: Valid container image URL used to reference a + remote bytecode image. + type: string + required: + - url + type: object + path: + description: Path is used to specify a bytecode object via filepath. + type: string + type: object + globaldata: + additionalProperties: + format: byte + type: string + description: |- + GlobalData allows the user to set global variables when the program is loaded + with an array of raw bytes. This is a very low level primitive. The caller + is responsible for formatting the byte string appropriately considering + such things as size, endianness, alignment and packing of data structures. + type: object + nodeselector: + description: |- + NodeSelector allows the user to specify which nodes to deploy the + bpf program to. This field must be specified, to select all nodes + use standard metav1.LabelSelector semantics and make it empty. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + programs: + description: |- + Programs is a list of bpf programs supported for a specific application. + It's possible that the application can selectively choose which program(s) + to run from this list. + items: + description: BpfApplicationProgram defines the desired state of + BpfApplication + properties: + fentry: + description: fentry defines the desired state of the application's + FentryPrograms. + properties: + bpffunctionname: + description: |- + BpfFunctionName is the name of the function that is the entry point for the BPF + program + type: string + func_name: + description: Function to attach the fentry to. + type: string + mapownerselector: + description: |- + MapOwnerSelector is used to select the loaded eBPF program this eBPF program + will share a map with. The value is a label applied to the BpfProgram to select. + The selector must resolve to exactly one instance of a BpfProgram on a given node + or the eBPF program will not load. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - bpffunctionname + - func_name + type: object + fexit: + description: fexit defines the desired state of the application's + FexitPrograms. + properties: + bpffunctionname: + description: |- + BpfFunctionName is the name of the function that is the entry point for the BPF + program + type: string + func_name: + description: Function to attach the fexit to. + type: string + mapownerselector: + description: |- + MapOwnerSelector is used to select the loaded eBPF program this eBPF program + will share a map with. The value is a label applied to the BpfProgram to select. + The selector must resolve to exactly one instance of a BpfProgram on a given node + or the eBPF program will not load. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - bpffunctionname + - func_name + type: object + kprobe: + description: kprobe defines the desired state of the application's + KprobePrograms. + properties: + bpffunctionname: + description: |- + BpfFunctionName is the name of the function that is the entry point for the BPF + program + type: string + func_name: + description: Functions to attach the kprobe to. + type: string + mapownerselector: + description: |- + MapOwnerSelector is used to select the loaded eBPF program this eBPF program + will share a map with. The value is a label applied to the BpfProgram to select. + The selector must resolve to exactly one instance of a BpfProgram on a given node + or the eBPF program will not load. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + offset: + default: 0 + description: |- + Offset added to the address of the function for kprobe. + Not allowed for kretprobes. + format: int64 + type: integer + retprobe: + default: false + description: Whether the program is a kretprobe. Default + is false + type: boolean + required: + - bpffunctionname + - func_name + type: object + kretprobe: + description: kretprobe defines the desired state of the application's + KretprobePrograms. + properties: + bpffunctionname: + description: |- + BpfFunctionName is the name of the function that is the entry point for the BPF + program + type: string + func_name: + description: Functions to attach the kprobe to. + type: string + mapownerselector: + description: |- + MapOwnerSelector is used to select the loaded eBPF program this eBPF program + will share a map with. The value is a label applied to the BpfProgram to select. + The selector must resolve to exactly one instance of a BpfProgram on a given node + or the eBPF program will not load. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + offset: + default: 0 + description: |- + Offset added to the address of the function for kprobe. + Not allowed for kretprobes. + format: int64 + type: integer + retprobe: + default: false + description: Whether the program is a kretprobe. Default + is false + type: boolean + required: + - bpffunctionname + - func_name + type: object + tc: + description: tc defines the desired state of the application's + TcPrograms. + properties: + bpffunctionname: + description: |- + BpfFunctionName is the name of the function that is the entry point for the BPF + program + type: string + direction: + description: |- + Direction specifies the direction of traffic the tc program should + attach to for a given network device. + enum: + - ingress + - egress + type: string + interfaceselector: + description: Selector to determine the network interface + (or interfaces) + maxProperties: 1 + minProperties: 1 + properties: + interfaces: + description: |- + Interfaces refers to a list of network interfaces to attach the BPF + program to. + items: + type: string + type: array + primarynodeinterface: + description: Attach BPF program to the primary interface + on the node. Only 'true' accepted. + type: boolean + type: object + mapownerselector: + description: |- + MapOwnerSelector is used to select the loaded eBPF program this eBPF program + will share a map with. The value is a label applied to the BpfProgram to select. + The selector must resolve to exactly one instance of a BpfProgram on a given node + or the eBPF program will not load. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + priority: + description: |- + Priority specifies the priority of the tc program in relation to + other programs of the same type with the same attach point. It is a value + from 0 to 1000 where lower values have higher precedence. + format: int32 + maximum: 1000 + minimum: 0 + type: integer + proceedon: + default: + - pipe + - dispatcher_return + description: |- + ProceedOn allows the user to call other tc programs in chain on this exit code. + Multiple values are supported by repeating the parameter. + items: + enum: + - unspec + - ok + - reclassify + - shot + - pipe + - stolen + - queued + - repeat + - redirect + - trap + - dispatcher_return + type: string + maxItems: 11 + type: array + required: + - bpffunctionname + - direction + - interfaceselector + - priority + type: object + tracepoint: + description: tracepoint defines the desired state of the application's + TracepointPrograms. + properties: + bpffunctionname: + description: |- + BpfFunctionName is the name of the function that is the entry point for the BPF + program + type: string + mapownerselector: + description: |- + MapOwnerSelector is used to select the loaded eBPF program this eBPF program + will share a map with. The value is a label applied to the BpfProgram to select. + The selector must resolve to exactly one instance of a BpfProgram on a given node + or the eBPF program will not load. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + names: + description: |- + Names refers to the names of kernel tracepoints to attach the + bpf program to. + items: + type: string + type: array + required: + - bpffunctionname + - names + type: object + type: + description: Type specifies the bpf program type + enum: + - XDP + - TC + - TCX + - Fentry + - Fexit + - Kprobe + - Kretprobe + - Uprobe + - Uretprobe + - Tracepoint + type: string + uprobe: + description: uprobe defines the desired state of the application's + UprobePrograms. + properties: + bpffunctionname: + description: |- + BpfFunctionName is the name of the function that is the entry point for the BPF + program + type: string + containers: + description: |- + Containers identifes the set of containers in which to attach the uprobe. + If Containers is not specified, the uprobe will be attached in the + bpfman-agent container. The ContainerSelector is very flexible and even + allows the selection of all containers in a cluster. If an attempt is + made to attach uprobes to too many containers, it can have a negative + impact on on the cluster. + properties: + containernames: + description: |- + Name(s) of container(s). If none are specified, all containers in the + pod are selected. + items: + type: string + type: array + namespace: + default: "" + description: Target namespaces. + type: string + pods: + description: |- + Target pods. This field must be specified, to select all pods use + standard metav1.LabelSelector semantics and make it empty. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - pods + type: object + func_name: + description: Function to attach the uprobe to. + type: string + mapownerselector: + description: |- + MapOwnerSelector is used to select the loaded eBPF program this eBPF program + will share a map with. The value is a label applied to the BpfProgram to select. + The selector must resolve to exactly one instance of a BpfProgram on a given node + or the eBPF program will not load. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + offset: + default: 0 + description: Offset added to the address of the function + for uprobe. + format: int64 + type: integer + pid: + description: |- + Only execute uprobe for given process identification number (PID). If PID + is not provided, uprobe executes for all PIDs. + format: int32 + type: integer + retprobe: + default: false + description: Whether the program is a uretprobe. Default + is false + type: boolean + target: + description: Library name or the absolute path to a binary + or library. + type: string + required: + - bpffunctionname + - target + type: object + uretprobe: + description: uretprobe defines the desired state of the application's + UretprobePrograms. + properties: + bpffunctionname: + description: |- + BpfFunctionName is the name of the function that is the entry point for the BPF + program + type: string + containers: + description: |- + Containers identifes the set of containers in which to attach the uprobe. + If Containers is not specified, the uprobe will be attached in the + bpfman-agent container. The ContainerSelector is very flexible and even + allows the selection of all containers in a cluster. If an attempt is + made to attach uprobes to too many containers, it can have a negative + impact on on the cluster. + properties: + containernames: + description: |- + Name(s) of container(s). If none are specified, all containers in the + pod are selected. + items: + type: string + type: array + namespace: + default: "" + description: Target namespaces. + type: string + pods: + description: |- + Target pods. This field must be specified, to select all pods use + standard metav1.LabelSelector semantics and make it empty. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - pods + type: object + func_name: + description: Function to attach the uprobe to. + type: string + mapownerselector: + description: |- + MapOwnerSelector is used to select the loaded eBPF program this eBPF program + will share a map with. The value is a label applied to the BpfProgram to select. + The selector must resolve to exactly one instance of a BpfProgram on a given node + or the eBPF program will not load. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + offset: + default: 0 + description: Offset added to the address of the function + for uprobe. + format: int64 + type: integer + pid: + description: |- + Only execute uprobe for given process identification number (PID). If PID + is not provided, uprobe executes for all PIDs. + format: int32 + type: integer + retprobe: + default: false + description: Whether the program is a uretprobe. Default + is false + type: boolean + target: + description: Library name or the absolute path to a binary + or library. + type: string + required: + - bpffunctionname + - target + type: object + xdp: + description: xdp defines the desired state of the application's + XdpPrograms. + properties: + bpffunctionname: + description: |- + BpfFunctionName is the name of the function that is the entry point for the BPF + program + type: string + interfaceselector: + description: Selector to determine the network interface + (or interfaces) + maxProperties: 1 + minProperties: 1 + properties: + interfaces: + description: |- + Interfaces refers to a list of network interfaces to attach the BPF + program to. + items: + type: string + type: array + primarynodeinterface: + description: Attach BPF program to the primary interface + on the node. Only 'true' accepted. + type: boolean + type: object + mapownerselector: + description: |- + MapOwnerSelector is used to select the loaded eBPF program this eBPF program + will share a map with. The value is a label applied to the BpfProgram to select. + The selector must resolve to exactly one instance of a BpfProgram on a given node + or the eBPF program will not load. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + priority: + description: |- + Priority specifies the priority of the bpf program in relation to + other programs of the same type with the same attach point. It is a value + from 0 to 1000 where lower values have higher precedence. + format: int32 + maximum: 1000 + minimum: 0 + type: integer + proceedon: + default: + - pass + - dispatcher_return + items: + enum: + - aborted + - drop + - pass + - tx + - redirect + - dispatcher_return + type: string + maxItems: 6 + type: array + required: + - bpffunctionname + - interfaceselector + - priority + type: object + type: object + minItems: 1 + type: array + required: + - bytecode + - nodeselector + type: object + status: + description: BpfApplicationStatus defines the observed state of BpfApplication + properties: + conditions: + description: |- + Conditions houses the global cluster state for the eBPFProgram. The explicit + condition types are defined internally. + items: + description: "Condition contains details for one aspect of the current + state of this API Resource.\n---\nThis struct is intended for + direct use as an array at the field path .status.conditions. For + example,\n\n\n\ttype FooStatus struct{\n\t // Represents the + observations of a foo's current state.\n\t // Known .status.conditions.type + are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // + +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t + \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t + \ // other fields\n\t}" + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: null + storedVersions: null diff --git a/config/bpfman-operator-deployment/kustomization.yaml b/config/bpfman-operator-deployment/kustomization.yaml index 816ab6601..d0eb3dd30 100644 --- a/config/bpfman-operator-deployment/kustomization.yaml +++ b/config/bpfman-operator-deployment/kustomization.yaml @@ -5,4 +5,4 @@ kind: Kustomization images: - name: quay.io/bpfman/bpfman-operator newName: quay.io/bpfman/bpfman-operator - newTag: latest-amd64 + newTag: latest diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index c6c0285af..849af96cc 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -7,5 +7,5 @@ resources: - bpfman.io_v1alpha1_uprobe_uprobeprogram.yaml - bpfman.io_v1alpha1_fentry_fentryprogram.yaml - bpfman.io_v1alpha1_fexit_fexitprogram.yaml -- _v1alpha1_bpfapplication.yaml + - _v1alpha1_bpfapplication.yaml # +kubebuilder:scaffold:manifestskustomizesamples From e02e40831884e6f0f93827f08877df8e80298d99 Mon Sep 17 00:00:00 2001 From: Andre Fredette Date: Thu, 6 Jun 2024 15:31:24 -0400 Subject: [PATCH 06/16] Additional changes for BpfApplication object - Ran make generate for latest BpfApplication CRD change - Handle setting owner and object type - If reconcile is complete for one program/object, continue with the next one - Added unit test for fentry TODO: - Add BpfApplication tests for all program types - Add sample BpfApplication yaml and test on kubernetes - See if there's any way to reduce code duplication in application-program.go Signed-off-by: Andre Fredette --- controllers/bpfman-agent/application-program.go | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/controllers/bpfman-agent/application-program.go b/controllers/bpfman-agent/application-program.go index c806dfe28..6f9f64f73 100644 --- a/controllers/bpfman-agent/application-program.go +++ b/controllers/bpfman-agent/application-program.go @@ -30,25 +30,16 @@ type BpfApplicationReconciler struct { // the bpfmanReconciler interface. We should think about what's needed and what // isn't. -// func (r *BpfApplicationReconciler) getFinalizer() string { -// return internal.BpfApplicationControllerFinalizer -// } -// func (r *BpfApplicationReconciler) getName() string { -// return r.currentApp.Name -// } func (r *BpfApplicationReconciler) getRecType() string { return internal.ApplicationString } -// func (r *BpfApplicationReconciler) getNode() *v1.Node { -// return r.ourNode -// } - -// func (r *BpfApplicationReconciler) getNodeSelector() *metav1.LabelSelector { -// return &r.currentApp.Spec.NodeSelector -// } +func (r *BpfApplicationReconciler) getNode() *v1.Node { + return r.ourNode +} +>>>>>>> 14b2026 (Additional changes for BpfApplication object) // func (r *BpfApplicationReconciler) getBpfGlobalData() map[string][]byte { // return r.currentApp.Spec.GlobalData From 96816e8d4ea78642342b3912e2904909f195530f Mon Sep 17 00:00:00 2001 From: Andre Fredette Date: Thu, 6 Jun 2024 15:31:24 -0400 Subject: [PATCH 07/16] Additional changes for BpfApplication object - Handle setting owner and object type - If reconcile is complete with one program/object, continue with the next one These changes have not been tested. Still need to write a unit test Signed-off-by: Andre Fredette --- controllers/bpfman-agent/application-program.go | 5 +++-- controllers/bpfman-agent/application-program_test.go | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/controllers/bpfman-agent/application-program.go b/controllers/bpfman-agent/application-program.go index 6f9f64f73..0579f9468 100644 --- a/controllers/bpfman-agent/application-program.go +++ b/controllers/bpfman-agent/application-program.go @@ -80,7 +80,9 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque var complete bool for _, a := range appPrograms.Items { + complete = false for _, p := range a.Spec.Programs { + complete = false switch p.Type { case bpfmaniov1alpha1.ProgTypeFentry: fentryProgram := bpfmaniov1alpha1.FentryProgram{ @@ -226,8 +228,7 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque complete, res, err = r.reconcileCommon(ctx, rec, xdpObjects) default: - r.Logger.Error(fmt.Errorf("Unsupported Bpf program type"), "Unsupported Bpf program type", "ProgType", p.Type) - // Skip this program and continue to the next one + r.Logger.Info("Unsupported Bpf program type", "ProgType", p.Type) continue } diff --git a/controllers/bpfman-agent/application-program_test.go b/controllers/bpfman-agent/application-program_test.go index 5d26ebba9..25324c783 100644 --- a/controllers/bpfman-agent/application-program_test.go +++ b/controllers/bpfman-agent/application-program_test.go @@ -66,7 +66,7 @@ func TestBpfApplicationControllerCreate(t *testing.T) { Type: bpfmaniov1alpha1.ProgTypeFentry, Fentry: &bpfmaniov1alpha1.FentryProgramInfo{ BpfProgramCommon: bpfmaniov1alpha1.BpfProgramCommon{ - BpfFunctionName: fentryBpfFunctionName, + BpfFunctionName: bpfFentryFunctionName, }, FunctionName: fentryFunctionName, }, @@ -75,7 +75,7 @@ func TestBpfApplicationControllerCreate(t *testing.T) { Type: bpfmaniov1alpha1.ProgTypeKprobe, Kprobe: &bpfmaniov1alpha1.KprobeProgramInfo{ BpfProgramCommon: bpfmaniov1alpha1.BpfProgramCommon{ - BpfFunctionName: kprobeBpfFunctionName, + BpfFunctionName: bpfKprobeFunctionName, }, FunctionName: kprobeFunctionName, Offset: uint64(kprobeOffset), @@ -86,7 +86,7 @@ func TestBpfApplicationControllerCreate(t *testing.T) { Type: bpfmaniov1alpha1.ProgTypeTracepoint, Tracepoint: &bpfmaniov1alpha1.TracepointProgramInfo{ BpfProgramCommon: bpfmaniov1alpha1.BpfProgramCommon{ - BpfFunctionName: tracepointBpfFunctionName, + BpfFunctionName: bpfTracepointFunctionName, }, Names: []string{tracepointName}, }, From e44d4f4775aefbc5ceddcb7ca4803841120a3d88 Mon Sep 17 00:00:00 2001 From: Andre Fredette Date: Sun, 9 Jun 2024 09:28:06 -0400 Subject: [PATCH 08/16] Add BpfParentProgram label to BpfPrograms When reconciling a *Program object created by an ApplicationProgram, getExistingBpfPrograms needs to return just the programs for the given *Program. However, the current implementation uses the BpfParentProgram label, so that gets all the BpfPrograms created for the ApplicationProgram, which messes up the reconcile logic. This commit adds another label called BpfParentProgram which allows us to get just the BpfPrograms for the current *Program regardless of whether they were created for a single *Program CRD or by a program entry for an ApplicationProgram CRD. Signed-off-by: Andre Fredette --- .../bpfman-agent/application-program.go | 72 +++++++++++++++---- controllers/bpfman-agent/common.go | 33 +++++---- controllers/bpfman-agent/fentry-program.go | 2 +- controllers/bpfman-agent/fexit-program.go | 2 +- controllers/bpfman-agent/kprobe-program.go | 2 +- controllers/bpfman-agent/tc-program.go | 2 +- .../bpfman-agent/tracepoint-program.go | 2 +- controllers/bpfman-agent/uprobe-program.go | 6 +- controllers/bpfman-agent/xdp-program.go | 2 +- internal/constants.go | 1 + 10 files changed, 88 insertions(+), 36 deletions(-) diff --git a/controllers/bpfman-agent/application-program.go b/controllers/bpfman-agent/application-program.go index 0579f9468..4985d3b44 100644 --- a/controllers/bpfman-agent/application-program.go +++ b/controllers/bpfman-agent/application-program.go @@ -30,8 +30,6 @@ type BpfApplicationReconciler struct { // the bpfmanReconciler interface. We should think about what's needed and what // isn't. - - func (r *BpfApplicationReconciler) getRecType() string { return internal.ApplicationString } @@ -39,11 +37,6 @@ func (r *BpfApplicationReconciler) getRecType() string { func (r *BpfApplicationReconciler) getNode() *v1.Node { return r.ourNode } ->>>>>>> 14b2026 (Additional changes for BpfApplication object) - -// func (r *BpfApplicationReconciler) getBpfGlobalData() map[string][]byte { -// return r.currentApp.Spec.GlobalData -// } func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { // Initialize node and current program @@ -79,10 +72,8 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque var err error var complete bool - for _, a := range appPrograms.Items { - complete = false - for _, p := range a.Spec.Programs { - complete = false + for i, a := range appPrograms.Items { + for j, p := range a.Spec.Programs { switch p.Type { case bpfmaniov1alpha1.ProgTypeFentry: fentryProgram := bpfmaniov1alpha1.FentryProgram{ @@ -104,6 +95,8 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque // Reconcile FentryProgram. complete, res, err = r.reconcileCommon(ctx, rec, fentryObjects) + r.showPrograms(ctx, rec) + case bpfmaniov1alpha1.ProgTypeFexit: fexitProgram := bpfmaniov1alpha1.FexitProgram{ ObjectMeta: metav1.ObjectMeta{ @@ -124,6 +117,8 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque // Reconcile FexitProgram. complete, res, err = r.reconcileCommon(ctx, rec, fexitObjects) + r.showPrograms(ctx, rec) + case bpfmaniov1alpha1.ProgTypeKprobe, bpfmaniov1alpha1.ProgTypeKretprobe: kprobeProgram := bpfmaniov1alpha1.KprobeProgram{ @@ -145,6 +140,8 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque // Reconcile KprobeProgram or KpretprobeProgram. complete, res, err = r.reconcileCommon(ctx, rec, kprobeObjects) + r.showPrograms(ctx, rec) + case bpfmaniov1alpha1.ProgTypeUprobe, bpfmaniov1alpha1.ProgTypeUretprobe: uprobeProgram := bpfmaniov1alpha1.UprobeProgram{ @@ -166,6 +163,8 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque // Reconcile UprobeProgram or UpretprobeProgram. complete, res, err = r.reconcileCommon(ctx, rec, uprobeObjects) + r.showPrograms(ctx, rec) + case bpfmaniov1alpha1.ProgTypeTracepoint: tracepointProgram := bpfmaniov1alpha1.TracepointProgram{ ObjectMeta: metav1.ObjectMeta{ @@ -186,6 +185,8 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque // Reconcile TracepointProgram. complete, res, err = r.reconcileCommon(ctx, rec, tracepointObjects) + r.showPrograms(ctx, rec) + case bpfmaniov1alpha1.ProgTypeTC, bpfmaniov1alpha1.ProgTypeTCX: tcProgram := bpfmaniov1alpha1.TcProgram{ @@ -207,6 +208,8 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque // Reconcile TcProgram. complete, res, err = r.reconcileCommon(ctx, rec, tcObjects) + r.showPrograms(ctx, rec) + case bpfmaniov1alpha1.ProgTypeXDP: xdpProgram := bpfmaniov1alpha1.XdpProgram{ ObjectMeta: metav1.ObjectMeta{ @@ -227,11 +230,19 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque // Reconcile XdpProgram. complete, res, err = r.reconcileCommon(ctx, rec, xdpObjects) + r.showPrograms(ctx, rec) + + r.showPrograms(ctx, rec) + default: - r.Logger.Info("Unsupported Bpf program type", "ProgType", p.Type) + r.Logger.Error(fmt.Errorf("unsupported bpf program type"), "unsupported bpf program type", "ProgType", p.Type) + // Skip this program and continue to the next one continue } + r.Logger.V(1).Info("Reconcile Application", "Application", i, "Program", j, "Name", a.Name, + "type", p.Type, "Complete", complete, "Result", res, "Error", err) + if complete { // We've completed reconciling this program, continue to the next one continue @@ -251,6 +262,43 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque return res, err } +// TODO: Remove this debug function +func (r *BpfApplicationReconciler) getAllExistingBpfPrograms(ctx context.Context, + rec bpfmanReconciler) (map[string]bpfmaniov1alpha1.BpfProgram, error) { + + bpfProgramList := &bpfmaniov1alpha1.BpfProgramList{} + + // Only list bpfPrograms for this *Program and the controller's node + opts := []client.ListOption{ + client.MatchingLabels{ + internal.BpfProgramOwnerLabel: rec.getOwner().GetName(), + internal.K8sHostLabel: r.NodeName, + }, + } + + err := r.List(ctx, bpfProgramList, opts...) + if err != nil { + return nil, err + } + + existingBpfPrograms := map[string]bpfmaniov1alpha1.BpfProgram{} + for _, bpfProg := range bpfProgramList.Items { + existingBpfPrograms[bpfProg.GetName()] = bpfProg + } + + return existingBpfPrograms, nil +} + +// TODO: Remove this debug function +func (r *BpfApplicationReconciler) showPrograms(ctx context.Context, rec bpfmanReconciler) { + programs, err := r.getAllExistingBpfPrograms(ctx, rec) + if err != nil { + r.Logger.V(1).Info("Failed to get existing BpfPrograms", "Application", rec.getOwner().GetName(), "Error", err) + } else { + r.Logger.V(1).Info("Existing BpfPrograms", "Application", rec.getOwner().GetName(), "Programs", programs) + } +} + // SetupWithManager sets up the controller with the Manager. // The Bpfman-Agent should reconcile whenever a BpfApplication object is updated, // load the programs to the node via bpfman, and then create a bpfProgram object diff --git a/controllers/bpfman-agent/common.go b/controllers/bpfman-agent/common.go index 124d1eaed..4e4b035a4 100644 --- a/controllers/bpfman-agent/common.go +++ b/controllers/bpfman-agent/common.go @@ -217,12 +217,10 @@ func (r *ReconcilerCommon) reconcileBpfProgram(ctx context.Context, if err != nil { return bpfmaniov1alpha1.BpfProgCondBytecodeSelectorError, err } - - r.Logger.V(1).WithValues("loadRequest", loadRequest).WithValues("loadedBpfProgram", loadedBpfProgram).Info("StateMatch") - isSame, reasons := bpfmanagentinternal.DoesProgExist(loadedBpfProgram, loadRequest) if !isSame { r.Logger.V(1).Info("bpf program is in wrong state, unloading and reloading", "reason", reasons, "bpfProgram Name", bpfProgram.Name, "bpf program ID", id) + r.Logger.V(1).WithValues("loadRequest", loadRequest).WithValues("loadedBpfProgram", loadedBpfProgram).Info("bpf program state") if err := bpfmanagentinternal.UnloadBpfmanProgram(ctx, r.BpfmanClient, *id); err != nil { r.Logger.Error(err, "Failed to unload BPF Program") return bpfmaniov1alpha1.BpfProgCondNotUnloaded, nil @@ -499,13 +497,17 @@ func (r *ReconcilerCommon) updateStatus(ctx context.Context, bpfProgram *bpfmani } func (r *ReconcilerCommon) getExistingBpfPrograms(ctx context.Context, - owner string) (map[string]bpfmaniov1alpha1.BpfProgram, error) { + rec bpfmanReconciler) (map[string]bpfmaniov1alpha1.BpfProgram, error) { bpfProgramList := &bpfmaniov1alpha1.BpfProgramList{} // Only list bpfPrograms for this *Program and the controller's node opts := []client.ListOption{ - client.MatchingLabels{internal.BpfProgramOwnerLabel: owner, internal.K8sHostLabel: r.NodeName}, + client.MatchingLabels{ + internal.BpfProgramOwnerLabel: rec.getOwner().GetName(), + internal.BpfParentProgram: rec.getName(), + internal.K8sHostLabel: r.NodeName, + }, } err := r.List(ctx, bpfProgramList, opts...) @@ -525,29 +527,30 @@ func (r *ReconcilerCommon) getExistingBpfPrograms(ctx context.Context, // into a central location. func (r *ReconcilerCommon) createBpfProgram( bpfProgramName string, - finalizer string, - owner metav1.Object, - ownerType string, + rec bpfmanReconciler, annotations map[string]string) (*bpfmaniov1alpha1.BpfProgram, error) { - r.Logger.V(1).Info("createBpfProgram()", "Name", bpfProgramName, "Owner", owner.GetName(), "OwnerType", ownerType) + r.Logger.V(1).Info("createBpfProgram()", "Name", bpfProgramName, + "Owner", rec.getOwner().GetName(), "OwnerType", rec.getRecType()) bpfProg := &bpfmaniov1alpha1.BpfProgram{ ObjectMeta: metav1.ObjectMeta{ Name: bpfProgramName, - Finalizers: []string{finalizer}, - Labels: map[string]string{internal.BpfProgramOwnerLabel: owner.GetName(), - internal.K8sHostLabel: r.NodeName}, + Finalizers: []string{rec.getFinalizer()}, + Labels: map[string]string{ + internal.BpfProgramOwnerLabel: rec.getOwner().GetName(), + internal.BpfParentProgram: rec.getName(), + internal.K8sHostLabel: r.NodeName}, Annotations: annotations, }, Spec: bpfmaniov1alpha1.BpfProgramSpec{ - Type: ownerType, + Type: rec.getRecType(), }, Status: bpfmaniov1alpha1.BpfProgramStatus{Conditions: []metav1.Condition{}}, } // Make the corresponding BpfProgramConfig the owner - if err := ctrl.SetControllerReference(owner, bpfProg, r.Scheme); err != nil { + if err := ctrl.SetControllerReference(rec.getOwner(), bpfProg, r.Scheme); err != nil { return nil, fmt.Errorf("failed to bpfProgram object owner reference: %v", err) } @@ -752,7 +755,7 @@ func (r *ReconcilerCommon) reconcileProgram(ctx context.Context, // Query the K8s API to get a list of existing bpfPrograms for this *Program // on this node. - existingBpfPrograms, err := r.getExistingBpfPrograms(ctx, rec.getOwner().GetName()) + existingBpfPrograms, err := r.getExistingBpfPrograms(ctx, rec) if err != nil { return internal.Requeue, fmt.Errorf("failed to get existing bpfPrograms: %v", err) } diff --git a/controllers/bpfman-agent/fentry-program.go b/controllers/bpfman-agent/fentry-program.go index eb72d3cc3..8e6883d87 100644 --- a/controllers/bpfman-agent/fentry-program.go +++ b/controllers/bpfman-agent/fentry-program.go @@ -134,7 +134,7 @@ func (r *FentryProgramReconciler) getExpectedBpfPrograms(ctx context.Context) (* annotations := map[string]string{internal.FentryProgramFunction: r.currentFentryProgram.Spec.FunctionName} - prog, err := r.createBpfProgram(bpfProgramName, r.getFinalizer(), r.getOwner(), r.getRecType(), annotations) + prog, err := r.createBpfProgram(bpfProgramName, r, annotations) if err != nil { return nil, fmt.Errorf("failed to create BpfProgram %s: %v", bpfProgramName, err) } diff --git a/controllers/bpfman-agent/fexit-program.go b/controllers/bpfman-agent/fexit-program.go index cde85e989..177a572c3 100644 --- a/controllers/bpfman-agent/fexit-program.go +++ b/controllers/bpfman-agent/fexit-program.go @@ -134,7 +134,7 @@ func (r *FexitProgramReconciler) getExpectedBpfPrograms(ctx context.Context) (*b annotations := map[string]string{internal.FexitProgramFunction: r.currentFexitProgram.Spec.FunctionName} - prog, err := r.createBpfProgram(bpfProgramName, r.getFinalizer(), r.getOwner(), r.getRecType(), annotations) + prog, err := r.createBpfProgram(bpfProgramName, r, annotations) if err != nil { return nil, fmt.Errorf("failed to create BpfProgram %s: %v", bpfProgramName, err) } diff --git a/controllers/bpfman-agent/kprobe-program.go b/controllers/bpfman-agent/kprobe-program.go index 5d0cd8123..340f77e97 100644 --- a/controllers/bpfman-agent/kprobe-program.go +++ b/controllers/bpfman-agent/kprobe-program.go @@ -134,7 +134,7 @@ func (r *KprobeProgramReconciler) getExpectedBpfPrograms(ctx context.Context) (* annotations := map[string]string{internal.KprobeProgramFunction: r.currentKprobeProgram.Spec.FunctionName} - prog, err := r.createBpfProgram(bpfProgramName, r.getFinalizer(), r.getOwner(), r.getRecType(), annotations) + prog, err := r.createBpfProgram(bpfProgramName, r, annotations) if err != nil { return nil, fmt.Errorf("failed to create BpfProgram %s: %v", bpfProgramName, err) } diff --git a/controllers/bpfman-agent/tc-program.go b/controllers/bpfman-agent/tc-program.go index f4fc403dc..751afe869 100644 --- a/controllers/bpfman-agent/tc-program.go +++ b/controllers/bpfman-agent/tc-program.go @@ -176,7 +176,7 @@ func (r *TcProgramReconciler) getExpectedBpfPrograms(ctx context.Context) (*bpfm bpfProgramName := fmt.Sprintf("%s-%s-%s", r.currentTcProgram.Name, r.NodeName, iface) annotations := map[string]string{internal.TcProgramInterface: iface} - prog, err := r.createBpfProgram(bpfProgramName, r.getFinalizer(), r.getOwner(), r.getRecType(), annotations) + prog, err := r.createBpfProgram(bpfProgramName, r, annotations) if err != nil { return nil, fmt.Errorf("failed to create BpfProgram %s: %v", bpfProgramName, err) } diff --git a/controllers/bpfman-agent/tracepoint-program.go b/controllers/bpfman-agent/tracepoint-program.go index 6e410e387..d583edf77 100644 --- a/controllers/bpfman-agent/tracepoint-program.go +++ b/controllers/bpfman-agent/tracepoint-program.go @@ -135,7 +135,7 @@ func (r *TracepointProgramReconciler) getExpectedBpfPrograms(ctx context.Context annotations := map[string]string{internal.TracepointProgramTracepoint: tracepoint} - prog, err := r.createBpfProgram(bpfProgramName, r.getFinalizer(), r.getOwner(), r.getRecType(), annotations) + prog, err := r.createBpfProgram(bpfProgramName, r, annotations) if err != nil { return nil, fmt.Errorf("failed to create BpfProgram %s: %v", bpfProgramName, err) } diff --git a/controllers/bpfman-agent/uprobe-program.go b/controllers/bpfman-agent/uprobe-program.go index 2fa428b7c..34c5bc5e6 100644 --- a/controllers/bpfman-agent/uprobe-program.go +++ b/controllers/bpfman-agent/uprobe-program.go @@ -183,7 +183,7 @@ func (r *UprobeProgramReconciler) getExpectedBpfPrograms(ctx context.Context) (* bpfProgramName := fmt.Sprintf("%s-%s", bpfProgramNameBase, "no-containers-on-node") - prog, err := r.createBpfProgram(bpfProgramName, r.getFinalizer(), r.getOwner(), r.getRecType(), annotations) + prog, err := r.createBpfProgram(bpfProgramName, r, annotations) if err != nil { return nil, fmt.Errorf("failed to create BpfProgram %s: %v", bpfProgramNameBase, err) } @@ -200,7 +200,7 @@ func (r *UprobeProgramReconciler) getExpectedBpfPrograms(ctx context.Context) (* bpfProgramName := fmt.Sprintf("%s-%s-%s", bpfProgramNameBase, container.podName, container.containerName) - prog, err := r.createBpfProgram(bpfProgramName, r.getFinalizer(), r.getOwner(), r.getRecType(), annotations) + prog, err := r.createBpfProgram(bpfProgramName, r, annotations) if err != nil { return nil, fmt.Errorf("failed to create BpfProgram %s: %v", bpfProgramName, err) } @@ -211,7 +211,7 @@ func (r *UprobeProgramReconciler) getExpectedBpfPrograms(ctx context.Context) (* } else { annotations := map[string]string{internal.UprobeProgramTarget: r.currentUprobeProgram.Spec.Target} - prog, err := r.createBpfProgram(bpfProgramNameBase, r.getFinalizer(), r.getOwner(), r.getRecType(), annotations) + prog, err := r.createBpfProgram(bpfProgramNameBase, r, annotations) if err != nil { return nil, fmt.Errorf("failed to create BpfProgram %s: %v", bpfProgramNameBase, err) } diff --git a/controllers/bpfman-agent/xdp-program.go b/controllers/bpfman-agent/xdp-program.go index 30d9a1526..ab45fd292 100644 --- a/controllers/bpfman-agent/xdp-program.go +++ b/controllers/bpfman-agent/xdp-program.go @@ -161,7 +161,7 @@ func (r *XdpProgramReconciler) getExpectedBpfPrograms(ctx context.Context) (*bpf bpfProgramName := fmt.Sprintf("%s-%s-%s", r.currentXdpProgram.Name, r.NodeName, iface) annotations := map[string]string{internal.XdpProgramInterface: iface} - prog, err := r.createBpfProgram(bpfProgramName, r.getFinalizer(), r.getOwner(), r.getRecType(), annotations) + prog, err := r.createBpfProgram(bpfProgramName, r, annotations) if err != nil { return nil, fmt.Errorf("failed to create BpfProgram %s: %v", bpfProgramName, err) } diff --git a/internal/constants.go b/internal/constants.go index a98317748..764ff2cd4 100644 --- a/internal/constants.go +++ b/internal/constants.go @@ -29,6 +29,7 @@ const ( FentryProgramFunction = "bpfman.io.fentryprogramcontroller/function" FexitProgramFunction = "bpfman.io.fexitprogramcontroller/function" BpfProgramOwnerLabel = "bpfman.io/ownedByProgram" + BpfParentProgram = "bpfman.io/parentProgram" K8sHostLabel = "kubernetes.io/hostname" DiscoveredLabel = "bpfman.io/discoveredProgram" IdAnnotation = "bpfman.io/ProgramId" From b0eda061af85d3db304b6c31537feae79054ab93 Mon Sep 17 00:00:00 2001 From: Mohamed Mahmoud Date: Sun, 9 Jun 2024 20:00:49 -0400 Subject: [PATCH 09/16] Add AppProgram agent controller unit-test and sample config Signed-off-by: Mohamed Mahmoud --- config/samples/_v1alpha1_bpfapplication.yaml | 12 -- .../bpfman.io_v1alpha1_bpfapplication.yaml | 34 +++++ config/samples/kustomization.yaml | 2 +- .../bpfman-agent/application-program.go | 4 - .../bpfman-agent/application-program_test.go | 124 +++++++++++++++--- 5 files changed, 140 insertions(+), 36 deletions(-) delete mode 100644 config/samples/_v1alpha1_bpfapplication.yaml create mode 100644 config/samples/bpfman.io_v1alpha1_bpfapplication.yaml diff --git a/config/samples/_v1alpha1_bpfapplication.yaml b/config/samples/_v1alpha1_bpfapplication.yaml deleted file mode 100644 index 3bb28d1d6..000000000 --- a/config/samples/_v1alpha1_bpfapplication.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: bpfman.io/v1alpha1 -kind: BpfApplication -metadata: - labels: - app.kubernetes.io/name: bpfapplication - app.kubernetes.io/instance: bpfapplication-sample - app.kubernetes.io/part-of: bpfman-operator - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/created-by: bpfman-operator - name: bpfapplication-sample -spec: - # TODO(user): Add fields here diff --git a/config/samples/bpfman.io_v1alpha1_bpfapplication.yaml b/config/samples/bpfman.io_v1alpha1_bpfapplication.yaml new file mode 100644 index 000000000..035f078d3 --- /dev/null +++ b/config/samples/bpfman.io_v1alpha1_bpfapplication.yaml @@ -0,0 +1,34 @@ +apiVersion: bpfman.io/v1alpha1 +kind: BpfApplication +metadata: + labels: + app.kubernetes.io/name: bpfapplication + name: bpfapplication-sample +spec: + # Select all nodes + nodeselector: {} + bytecode: + image: + url: quay.io/bpfman-bytecode/testapp:latest + globaldata: + GLOBAL_u8: + - 0x01 + GLOBAL_u32: + - 0x0D + - 0x0C + - 0x0B + - 0x0A + programs: + - type: Fentry + fentry: + func_name: do_unlinkat + - type: Kprobe + kprobe: + bpffunctionname: my_kprobe + func_name: try_to_wake_up + offset: 0 + retprobe: false + - type: Tracepoint + tracepoint: + names: + - syscalls/sys_enter_openat diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index 849af96cc..4795266f4 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -7,5 +7,5 @@ resources: - bpfman.io_v1alpha1_uprobe_uprobeprogram.yaml - bpfman.io_v1alpha1_fentry_fentryprogram.yaml - bpfman.io_v1alpha1_fexit_fexitprogram.yaml - - _v1alpha1_bpfapplication.yaml + - bpfman.io_v1alpha1_bpfapplication.yaml # +kubebuilder:scaffold:manifestskustomizesamples diff --git a/controllers/bpfman-agent/application-program.go b/controllers/bpfman-agent/application-program.go index 4985d3b44..a7bbe36b3 100644 --- a/controllers/bpfman-agent/application-program.go +++ b/controllers/bpfman-agent/application-program.go @@ -34,10 +34,6 @@ func (r *BpfApplicationReconciler) getRecType() string { return internal.ApplicationString } -func (r *BpfApplicationReconciler) getNode() *v1.Node { - return r.ourNode -} - func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { // Initialize node and current program r.currentApp = &bpfmaniov1alpha1.BpfApplication{} diff --git a/controllers/bpfman-agent/application-program_test.go b/controllers/bpfman-agent/application-program_test.go index 25324c783..73311fd95 100644 --- a/controllers/bpfman-agent/application-program_test.go +++ b/controllers/bpfman-agent/application-program_test.go @@ -34,19 +34,20 @@ func TestBpfApplicationControllerCreate(t *testing.T) { fakeNode = testutils.NewNode("fake-control-plane") ctx = context.TODO() // fentry program config - fentryBpfFunctionName = "fentry_test" + bpfFentryFunctionName = "fentry_test" fentryFunctionName = "do_unlinkat" fentryBpfProgName = fmt.Sprintf("%s-%s-%s-%s", name, "fentry", fakeNode.Name, "do-unlinkat") fentryBpfProg = &bpfmaniov1alpha1.BpfProgram{} fentryFakeUID = "ef71d42c-aa21-48e8-a697-82391d801a81" // kprobe program config - kprobeBpfFunctionName = "kprobe_test" - kprobeFunctionName = "try_to_wake_up" - kprobeOffset = 0 - kprobeRetprobe = false - // tracepoint program config - tracepointBpfFunctionName = "tracepoint_test" - tracepointName = "syscalls/sys_enter_setitimer" + bpfKprobeFunctionName = "kprobe_test" + kprobeFunctionName = "try_to_wake_up" + kprobeBpfProgName = fmt.Sprintf("%s-%s-%s-%s", name, "kprobe", fakeNode.Name, "try-to-wake-up") + kprobeBpfProg = &bpfmaniov1alpha1.BpfProgram{} + kprobeFakeUID = "ef71d42c-aa21-48e8-a697-82391d801a82" + kprobeOffset = 0 + kprobeRetprobe = false + kprobecontainerpid int32 = 0 ) // A AppProgram object with metadata and spec. @@ -82,15 +83,6 @@ func TestBpfApplicationControllerCreate(t *testing.T) { RetProbe: kprobeRetprobe, }, }, - { - Type: bpfmaniov1alpha1.ProgTypeTracepoint, - Tracepoint: &bpfmaniov1alpha1.TracepointProgramInfo{ - BpfProgramCommon: bpfmaniov1alpha1.BpfProgramCommon{ - BpfFunctionName: bpfTracepointFunctionName, - }, - Names: []string{tracepointName}, - }, - }, }, }, } @@ -106,6 +98,7 @@ func TestBpfApplicationControllerCreate(t *testing.T) { s.AddKnownTypes(bpfmaniov1alpha1.SchemeGroupVersion, &bpfmaniov1alpha1.BpfProgramList{}) s.AddKnownTypes(bpfmaniov1alpha1.SchemeGroupVersion, &bpfmaniov1alpha1.BpfProgram{}) s.AddKnownTypes(bpfmaniov1alpha1.SchemeGroupVersion, &bpfmaniov1alpha1.FentryProgramList{}) + s.AddKnownTypes(bpfmaniov1alpha1.SchemeGroupVersion, &bpfmaniov1alpha1.KprobeProgramList{}) // Create a fake client to mock API calls. cl := fake.NewClientBuilder().WithStatusSubresource(App).WithStatusSubresource(&bpfmaniov1alpha1.BpfProgram{}).WithRuntimeObjects(objs...).Build() @@ -120,7 +113,7 @@ func TestBpfApplicationControllerCreate(t *testing.T) { appOwner: App, } - // Set development Logger so we can see all logs in tests. + // Set development Logger, so we can see all logs in tests. logf.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{Development: true}))) // Create a ReconcileMemcached object with the scheme and fake client. @@ -135,6 +128,7 @@ func TestBpfApplicationControllerCreate(t *testing.T) { }, } + // do fentry program // First reconcile should create the bpf program object res, err := r.Reconcile(ctx, req) if err != nil { @@ -173,11 +167,13 @@ func TestBpfApplicationControllerCreate(t *testing.T) { // Require no requeue require.False(t, res.Requeue) + + // 1- do Fentry Program expectedLoadReq := &gobpfman.LoadRequest{ Bytecode: &gobpfman.BytecodeLocation{ Location: &gobpfman.BytecodeLocation_File{File: bytecodePath}, }, - Name: fentryBpfFunctionName, + Name: bpfFentryFunctionName, ProgramType: *internal.Tracing.Uint32(), Metadata: map[string]string{internal.UuidMetadataKey: string(fentryBpfProg.UID), internal.ProgramNameKey: name}, MapOwnerId: nil, @@ -219,4 +215,94 @@ func TestBpfApplicationControllerCreate(t *testing.T) { require.NoError(t, err) require.Equal(t, string(bpfmaniov1alpha1.BpfProgCondLoaded), fentryBpfProg.Status.Conditions[0].Type) + + // do kprobe program + // First reconcile should create the bpf program object + res, err = r.Reconcile(ctx, req) + if err != nil { + t.Fatalf("reconcile: (%v)", err) + } + + err = cl.Get(ctx, types.NamespacedName{Name: kprobeBpfProgName, Namespace: metav1.NamespaceAll}, kprobeBpfProg) + require.NoError(t, err) + + require.NotEmpty(t, kprobeBpfProg) + // Finalizer is written + require.Equal(t, internal.KprobeProgramControllerFinalizer, kprobeBpfProg.Finalizers[0]) + // owningConfig Label was correctly set + require.Equal(t, name, kprobeBpfProg.Labels[internal.BpfProgramOwnerLabel]) + // node Label was correctly set + require.Equal(t, fakeNode.Name, kprobeBpfProg.Labels[internal.K8sHostLabel]) + // fentry function Annotation was correctly set + require.Equal(t, kprobeFunctionName, kprobeBpfProg.Annotations[internal.KprobeProgramFunction]) + // Type is set + require.Equal(t, r.getRecType(), kprobeBpfProg.Spec.Type) + // Require no requeue + require.False(t, res.Requeue) + + // Update UID of bpfProgram with Fake UID since the fake API server won't + kprobeBpfProg.UID = types.UID(kprobeFakeUID) + err = cl.Update(ctx, kprobeBpfProg) + require.NoError(t, err) + + // Second reconcile should create the bpfman Load Request and update the + // BpfProgram object's maps field and id annotation. + res, err = r.Reconcile(ctx, req) + if err != nil { + t.Fatalf("reconcile: (%v)", err) + } + + // Require no requeue + require.False(t, res.Requeue) + + expectedLoadReq = &gobpfman.LoadRequest{ + Bytecode: &gobpfman.BytecodeLocation{ + Location: &gobpfman.BytecodeLocation_File{File: bytecodePath}, + }, + Name: bpfKprobeFunctionName, + ProgramType: *internal.Kprobe.Uint32(), + Metadata: map[string]string{internal.UuidMetadataKey: string(kprobeBpfProg.UID), internal.ProgramNameKey: name}, + MapOwnerId: nil, + Attach: &gobpfman.AttachInfo{ + Info: &gobpfman.AttachInfo_KprobeAttachInfo{ + KprobeAttachInfo: &gobpfman.KprobeAttachInfo{ + FnName: kprobeFunctionName, + Offset: uint64(kprobeOffset), + Retprobe: kprobeRetprobe, + ContainerPid: &kprobecontainerpid, + }, + }, + }, + } + + // Check that the bpfProgram's programs was correctly updated + err = cl.Get(ctx, types.NamespacedName{Name: kprobeBpfProgName, Namespace: metav1.NamespaceAll}, kprobeBpfProg) + require.NoError(t, err) + + // prog ID should already have been set + id, err = bpfmanagentinternal.GetID(kprobeBpfProg) + require.NoError(t, err) + + // Check the bpfLoadRequest was correctly Built + if !cmp.Equal(expectedLoadReq, cli.LoadRequests[int(*id)], protocmp.Transform()) { + cmp.Diff(expectedLoadReq, cli.LoadRequests[int(*id)], protocmp.Transform()) + t.Logf("Diff %v", cmp.Diff(expectedLoadReq, cli.LoadRequests[int(*id)], protocmp.Transform())) + t.Fatal("Built bpfman LoadRequest does not match expected") + } + + // Third reconcile should set the status to loaded + res, err = r.Reconcile(ctx, req) + if err != nil { + t.Fatalf("reconcile: (%v)", err) + } + + // Require no requeue + require.False(t, res.Requeue) + + // Check that the bpfProgram's status was correctly updated + err = cl.Get(ctx, types.NamespacedName{Name: kprobeBpfProgName, Namespace: metav1.NamespaceAll}, kprobeBpfProg) + require.NoError(t, err) + + require.Equal(t, string(bpfmaniov1alpha1.BpfProgCondLoaded), kprobeBpfProg.Status.Conditions[0].Type) + } From e8d57d18d87eae545bc4c3e3fa090e96457ae504 Mon Sep 17 00:00:00 2001 From: Andre Fredette Date: Mon, 10 Jun 2024 20:37:12 -0400 Subject: [PATCH 10/16] BpfApplication operator controller needs to watch BpfPrograms it owns Signed-off-by: Andre Fredette --- controllers/bpfman-operator/application-programs.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/controllers/bpfman-operator/application-programs.go b/controllers/bpfman-operator/application-programs.go index 64998a9b7..1c96adcf6 100644 --- a/controllers/bpfman-operator/application-programs.go +++ b/controllers/bpfman-operator/application-programs.go @@ -26,7 +26,10 @@ import ( "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" bpfmaniov1alpha1 "github.com/bpfman/bpfman-operator/apis/v1alpha1" internal "github.com/bpfman/bpfman-operator/internal" @@ -94,6 +97,12 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque func (r *BpfApplicationReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&bpfmaniov1alpha1.BpfApplication{}). + // Watch bpfPrograms which are owned by BpfApplications + Watches( + &bpfmaniov1alpha1.BpfProgram{}, + &handler.EnqueueRequestForObject{}, + builder.WithPredicates(predicate.And(statusChangedPredicate(), internal.BpfProgramTypePredicate(internal.ApplicationString))), + ). Complete(r) } From 67b5f6ae20320449bd05e10515b4d8b4d05dda6e Mon Sep 17 00:00:00 2001 From: Andre Fredette Date: Mon, 10 Jun 2024 20:49:18 -0400 Subject: [PATCH 11/16] Display status column for BpfApplications Signed-off-by: Andre Fredette --- apis/v1alpha1/bpfapplication_types.go | 1 + 1 file changed, 1 insertion(+) diff --git a/apis/v1alpha1/bpfapplication_types.go b/apis/v1alpha1/bpfapplication_types.go index 6bc1fbcc5..f6a8ad4dd 100644 --- a/apis/v1alpha1/bpfapplication_types.go +++ b/apis/v1alpha1/bpfapplication_types.go @@ -133,6 +133,7 @@ type BpfApplicationStatus struct { //+kubebuilder:resource:scope=Cluster // BpfApplication is the Schema for the bpfapplications API +// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.conditions[0].reason` type BpfApplication struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` From e5ee05b2d251fe4df88ad6d238f239ac1a669a60 Mon Sep 17 00:00:00 2001 From: Andre Fredette Date: Tue, 11 Jun 2024 12:00:31 -0400 Subject: [PATCH 12/16] Fixes to get BpfApplication delete working Two changes were needed: - The BpfPrograms needed the internal.BpfApplicationControllerFinalizer instead of the *Program finalizer. - GetDeletionTimestamp() needs to be called on the Owner. In this case, the owner is the BpfApplication. Also moved SetupWithManager() in the BpfApplication operator controller so that it was in the same order as the other controllers. This made a diff easier to read. Signed-off-by: Andre Fredette --- .../kustomization.yaml | 2 +- .../crd/bases/bpfman.io_bpfapplications.yaml | 6 +++- .../bpfman-agent/application-program.go | 2 ++ .../bpfman-agent/application-program_test.go | 4 +-- controllers/bpfman-agent/common.go | 4 ++- controllers/bpfman-agent/fentry-program.go | 10 +++---- controllers/bpfman-agent/fexit-program.go | 10 +++---- controllers/bpfman-agent/kprobe-program.go | 10 +++---- controllers/bpfman-agent/tc-program.go | 10 +++---- .../bpfman-agent/tracepoint-program.go | 10 +++---- controllers/bpfman-agent/uprobe-program.go | 10 +++---- controllers/bpfman-agent/xdp-program.go | 11 ++++---- .../bpfman-operator/application-programs.go | 28 ++++++++++--------- controllers/bpfman-operator/common.go | 2 +- 14 files changed, 58 insertions(+), 61 deletions(-) diff --git a/config/bpfman-operator-deployment/kustomization.yaml b/config/bpfman-operator-deployment/kustomization.yaml index d0eb3dd30..816ab6601 100644 --- a/config/bpfman-operator-deployment/kustomization.yaml +++ b/config/bpfman-operator-deployment/kustomization.yaml @@ -5,4 +5,4 @@ kind: Kustomization images: - name: quay.io/bpfman/bpfman-operator newName: quay.io/bpfman/bpfman-operator - newTag: latest + newTag: latest-amd64 diff --git a/config/crd/bases/bpfman.io_bpfapplications.yaml b/config/crd/bases/bpfman.io_bpfapplications.yaml index ea865814c..3e27e59d1 100644 --- a/config/crd/bases/bpfman.io_bpfapplications.yaml +++ b/config/crd/bases/bpfman.io_bpfapplications.yaml @@ -14,7 +14,11 @@ spec: singular: bpfapplication scope: Cluster versions: - - name: v1alpha1 + - additionalPrinterColumns: + - jsonPath: .status.conditions[0].reason + name: Status + type: string + name: v1alpha1 schema: openAPIV3Schema: description: BpfApplication is the Schema for the bpfapplications API diff --git a/controllers/bpfman-agent/application-program.go b/controllers/bpfman-agent/application-program.go index a7bbe36b3..b5e94a6fe 100644 --- a/controllers/bpfman-agent/application-program.go +++ b/controllers/bpfman-agent/application-program.go @@ -40,6 +40,8 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque r.ourNode = &v1.Node{} r.Logger = ctrl.Log.WithName("application") r.appOwner = &bpfmaniov1alpha1.BpfApplication{} + r.finalizer = internal.BpfApplicationControllerFinalizer + r.recType = internal.ApplicationString ctxLogger := log.FromContext(ctx) ctxLogger.Info("Reconcile Application: Enter", "ReconcileKey", req) diff --git a/controllers/bpfman-agent/application-program_test.go b/controllers/bpfman-agent/application-program_test.go index 73311fd95..bdd61a4f9 100644 --- a/controllers/bpfman-agent/application-program_test.go +++ b/controllers/bpfman-agent/application-program_test.go @@ -141,7 +141,7 @@ func TestBpfApplicationControllerCreate(t *testing.T) { require.NotEmpty(t, fentryBpfProg) // Finalizer is written - require.Equal(t, internal.FentryProgramControllerFinalizer, fentryBpfProg.Finalizers[0]) + require.Equal(t, internal.BpfApplicationControllerFinalizer, fentryBpfProg.Finalizers[0]) // owningConfig Label was correctly set require.Equal(t, name, fentryBpfProg.Labels[internal.BpfProgramOwnerLabel]) // node Label was correctly set @@ -228,7 +228,7 @@ func TestBpfApplicationControllerCreate(t *testing.T) { require.NotEmpty(t, kprobeBpfProg) // Finalizer is written - require.Equal(t, internal.KprobeProgramControllerFinalizer, kprobeBpfProg.Finalizers[0]) + require.Equal(t, internal.BpfApplicationControllerFinalizer, kprobeBpfProg.Finalizers[0]) // owningConfig Label was correctly set require.Equal(t, name, kprobeBpfProg.Labels[internal.BpfProgramOwnerLabel]) // node Label was correctly set diff --git a/controllers/bpfman-agent/common.go b/controllers/bpfman-agent/common.go index 4e4b035a4..91e592f80 100644 --- a/controllers/bpfman-agent/common.go +++ b/controllers/bpfman-agent/common.go @@ -72,6 +72,8 @@ type ReconcilerCommon struct { Logger logr.Logger NodeName string progId *uint32 + finalizer string + recType string appOwner metav1.Object // Set if the owner is an application } @@ -751,7 +753,7 @@ func (r *ReconcilerCommon) reconcileProgram(ctx context.Context, return internal.Requeue, fmt.Errorf("failed to check if node is selected: %v", err) } - isBeingDeleted := !program.GetDeletionTimestamp().IsZero() + isBeingDeleted := !rec.getOwner().GetDeletionTimestamp().IsZero() // Query the K8s API to get a list of existing bpfPrograms for this *Program // on this node. diff --git a/controllers/bpfman-agent/fentry-program.go b/controllers/bpfman-agent/fentry-program.go index 8e6883d87..f402d5cad 100644 --- a/controllers/bpfman-agent/fentry-program.go +++ b/controllers/bpfman-agent/fentry-program.go @@ -47,7 +47,7 @@ type FentryProgramReconciler struct { } func (r *FentryProgramReconciler) getFinalizer() string { - return internal.FentryProgramControllerFinalizer + return r.finalizer } func (r *FentryProgramReconciler) getOwner() metav1.Object { @@ -59,11 +59,7 @@ func (r *FentryProgramReconciler) getOwner() metav1.Object { } func (r *FentryProgramReconciler) getRecType() string { - if r.appOwner == nil { - return internal.FentryString - } else { - return internal.ApplicationString - } + return r.recType } func (r *FentryProgramReconciler) getProgType() internal.ProgramType { @@ -147,6 +143,8 @@ func (r *FentryProgramReconciler) getExpectedBpfPrograms(ctx context.Context) (* func (r *FentryProgramReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { // Initialize node and current program r.currentFentryProgram = &bpfmaniov1alpha1.FentryProgram{} + r.finalizer = internal.FentryProgramControllerFinalizer + r.recType = internal.FentryString r.ourNode = &v1.Node{} r.Logger = ctrl.Log.WithName("fentry") diff --git a/controllers/bpfman-agent/fexit-program.go b/controllers/bpfman-agent/fexit-program.go index 177a572c3..7e92f6e8f 100644 --- a/controllers/bpfman-agent/fexit-program.go +++ b/controllers/bpfman-agent/fexit-program.go @@ -47,7 +47,7 @@ type FexitProgramReconciler struct { } func (r *FexitProgramReconciler) getFinalizer() string { - return internal.FexitProgramControllerFinalizer + return r.finalizer } func (r *FexitProgramReconciler) getOwner() metav1.Object { @@ -59,11 +59,7 @@ func (r *FexitProgramReconciler) getOwner() metav1.Object { } func (r *FexitProgramReconciler) getRecType() string { - if r.appOwner == nil { - return internal.FexitString - } else { - return internal.ApplicationString - } + return r.recType } func (r *FexitProgramReconciler) getProgType() internal.ProgramType { @@ -147,6 +143,8 @@ func (r *FexitProgramReconciler) getExpectedBpfPrograms(ctx context.Context) (*b func (r *FexitProgramReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { // Initialize node and current program r.currentFexitProgram = &bpfmaniov1alpha1.FexitProgram{} + r.finalizer = internal.FexitProgramControllerFinalizer + r.recType = internal.FexitString r.ourNode = &v1.Node{} r.Logger = ctrl.Log.WithName("fexit") diff --git a/controllers/bpfman-agent/kprobe-program.go b/controllers/bpfman-agent/kprobe-program.go index 340f77e97..7e79dc7e8 100644 --- a/controllers/bpfman-agent/kprobe-program.go +++ b/controllers/bpfman-agent/kprobe-program.go @@ -47,7 +47,7 @@ type KprobeProgramReconciler struct { } func (r *KprobeProgramReconciler) getFinalizer() string { - return internal.KprobeProgramControllerFinalizer + return r.finalizer } func (r *KprobeProgramReconciler) getOwner() metav1.Object { @@ -59,11 +59,7 @@ func (r *KprobeProgramReconciler) getOwner() metav1.Object { } func (r *KprobeProgramReconciler) getRecType() string { - if r.appOwner == nil { - return internal.Kprobe.String() - } else { - return internal.ApplicationString - } + return r.recType } func (r *KprobeProgramReconciler) getProgType() internal.ProgramType { @@ -147,6 +143,8 @@ func (r *KprobeProgramReconciler) getExpectedBpfPrograms(ctx context.Context) (* func (r *KprobeProgramReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { // Initialize node and current program r.currentKprobeProgram = &bpfmaniov1alpha1.KprobeProgram{} + r.finalizer = internal.KprobeProgramControllerFinalizer + r.recType = internal.Kprobe.String() r.ourNode = &v1.Node{} r.Logger = ctrl.Log.WithName("kprobe") diff --git a/controllers/bpfman-agent/tc-program.go b/controllers/bpfman-agent/tc-program.go index 751afe869..f1499937e 100644 --- a/controllers/bpfman-agent/tc-program.go +++ b/controllers/bpfman-agent/tc-program.go @@ -48,7 +48,7 @@ type TcProgramReconciler struct { } func (r *TcProgramReconciler) getFinalizer() string { - return internal.TcProgramControllerFinalizer + return r.finalizer } func (r *TcProgramReconciler) getOwner() metav1.Object { @@ -60,11 +60,7 @@ func (r *TcProgramReconciler) getOwner() metav1.Object { } func (r *TcProgramReconciler) getRecType() string { - if r.appOwner == nil { - return internal.Tc.String() - } else { - return internal.ApplicationString - } + return r.recType } func (r *TcProgramReconciler) getProgType() internal.ProgramType { @@ -190,6 +186,8 @@ func (r *TcProgramReconciler) getExpectedBpfPrograms(ctx context.Context) (*bpfm func (r *TcProgramReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { // Initialize node and current program r.currentTcProgram = &bpfmaniov1alpha1.TcProgram{} + r.finalizer = internal.TcProgramControllerFinalizer + r.recType = internal.Tc.String() r.ourNode = &v1.Node{} r.Logger = ctrl.Log.WithName("tc") diff --git a/controllers/bpfman-agent/tracepoint-program.go b/controllers/bpfman-agent/tracepoint-program.go index d583edf77..5d701d07d 100644 --- a/controllers/bpfman-agent/tracepoint-program.go +++ b/controllers/bpfman-agent/tracepoint-program.go @@ -47,7 +47,7 @@ type TracepointProgramReconciler struct { } func (r *TracepointProgramReconciler) getFinalizer() string { - return internal.TracepointProgramControllerFinalizer + return r.finalizer } func (r *TracepointProgramReconciler) getOwner() metav1.Object { @@ -59,11 +59,7 @@ func (r *TracepointProgramReconciler) getOwner() metav1.Object { } func (r *TracepointProgramReconciler) getRecType() string { - if r.appOwner == nil { - return internal.Tracepoint.String() - } else { - return internal.ApplicationString - } + return r.recType } func (r *TracepointProgramReconciler) getProgType() internal.ProgramType { @@ -149,6 +145,8 @@ func (r *TracepointProgramReconciler) getExpectedBpfPrograms(ctx context.Context func (r *TracepointProgramReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { // Initialize node and current program r.currentTracepointProgram = &bpfmaniov1alpha1.TracepointProgram{} + r.finalizer = internal.TracepointProgramControllerFinalizer + r.recType = internal.Tracepoint.String() r.ourNode = &v1.Node{} r.Logger = ctrl.Log.WithName("tracept") diff --git a/controllers/bpfman-agent/uprobe-program.go b/controllers/bpfman-agent/uprobe-program.go index 34c5bc5e6..ebe7e4929 100644 --- a/controllers/bpfman-agent/uprobe-program.go +++ b/controllers/bpfman-agent/uprobe-program.go @@ -48,7 +48,7 @@ type UprobeProgramReconciler struct { } func (r *UprobeProgramReconciler) getFinalizer() string { - return internal.UprobeProgramControllerFinalizer + return r.finalizer } func (r *UprobeProgramReconciler) getOwner() metav1.Object { @@ -60,11 +60,7 @@ func (r *UprobeProgramReconciler) getOwner() metav1.Object { } func (r *UprobeProgramReconciler) getRecType() string { - if r.appOwner == nil { - return internal.UprobeString - } else { - return internal.ApplicationString - } + return r.recType } func (r *UprobeProgramReconciler) getProgType() internal.ProgramType { @@ -225,6 +221,8 @@ func (r *UprobeProgramReconciler) getExpectedBpfPrograms(ctx context.Context) (* func (r *UprobeProgramReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { // Initialize node and current program r.currentUprobeProgram = &bpfmaniov1alpha1.UprobeProgram{} + r.finalizer = internal.UprobeProgramControllerFinalizer + r.recType = internal.UprobeString r.ourNode = &v1.Node{} r.Logger = ctrl.Log.WithName("uprobe") diff --git a/controllers/bpfman-agent/xdp-program.go b/controllers/bpfman-agent/xdp-program.go index ab45fd292..88afccfe7 100644 --- a/controllers/bpfman-agent/xdp-program.go +++ b/controllers/bpfman-agent/xdp-program.go @@ -47,7 +47,7 @@ type XdpProgramReconciler struct { } func (r *XdpProgramReconciler) getFinalizer() string { - return internal.XdpProgramControllerFinalizer + return r.finalizer } func (r *XdpProgramReconciler) getOwner() metav1.Object { @@ -59,12 +59,9 @@ func (r *XdpProgramReconciler) getOwner() metav1.Object { } func (r *XdpProgramReconciler) getRecType() string { - if r.appOwner == nil { - return internal.Xdp.String() - } else { - return internal.ApplicationString - } + return r.recType } + func (r *XdpProgramReconciler) getProgType() internal.ProgramType { return internal.Xdp } @@ -175,6 +172,8 @@ func (r *XdpProgramReconciler) getExpectedBpfPrograms(ctx context.Context) (*bpf func (r *XdpProgramReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { // Initialize node and current program r.currentXdpProgram = &bpfmaniov1alpha1.XdpProgram{} + r.finalizer = internal.XdpProgramControllerFinalizer + r.recType = internal.Xdp.String() r.ourNode = &v1.Node{} r.Logger = ctrl.Log.WithName("xdp") diff --git a/controllers/bpfman-operator/application-programs.go b/controllers/bpfman-operator/application-programs.go index 1c96adcf6..0001c3430 100644 --- a/controllers/bpfman-operator/application-programs.go +++ b/controllers/bpfman-operator/application-programs.go @@ -52,6 +52,19 @@ func (r *BpfApplicationReconciler) getFinalizer() string { return internal.BpfApplicationControllerFinalizer } +// SetupWithManager sets up the controller with the Manager. +func (r *BpfApplicationReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&bpfmaniov1alpha1.BpfApplication{}). + // Watch bpfPrograms which are owned by BpfApplications + Watches( + &bpfmaniov1alpha1.BpfProgram{}, + &handler.EnqueueRequestForObject{}, + builder.WithPredicates(predicate.And(statusChangedPredicate(), internal.BpfProgramTypePredicate(internal.ApplicationString))), + ). + Complete(r) +} + func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { r.Logger = log.FromContext(ctx) @@ -93,20 +106,9 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque return reconcileBpfProgram(ctx, r, appProgram) } -// SetupWithManager sets up the controller with the Manager. -func (r *BpfApplicationReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&bpfmaniov1alpha1.BpfApplication{}). - // Watch bpfPrograms which are owned by BpfApplications - Watches( - &bpfmaniov1alpha1.BpfProgram{}, - &handler.EnqueueRequestForObject{}, - builder.WithPredicates(predicate.And(statusChangedPredicate(), internal.BpfProgramTypePredicate(internal.ApplicationString))), - ). - Complete(r) -} - func (r *BpfApplicationReconciler) updateStatus(ctx context.Context, name string, cond bpfmaniov1alpha1.ProgramConditionType, message string) (ctrl.Result, error) { + // Sometimes we end up with a stale FentryProgram due to races, do this + // get to ensure we're up to date before attempting a status update. app := &bpfmaniov1alpha1.BpfApplication{} if err := r.Get(ctx, types.NamespacedName{Namespace: corev1.NamespaceAll, Name: name}, app); err != nil { r.Logger.V(1).Info("failed to get fresh Application Programs object...requeuing") diff --git a/controllers/bpfman-operator/common.go b/controllers/bpfman-operator/common.go index fccec2792..2bcef68f3 100644 --- a/controllers/bpfman-operator/common.go +++ b/controllers/bpfman-operator/common.go @@ -161,7 +161,7 @@ func (r *ReconcilerCommon) removeFinalizer(ctx context.Context, prog client.Obje } func (r *ReconcilerCommon) addFinalizer(ctx context.Context, prog client.Object, finalizer string) (ctrl.Result, error) { - controllerutil.AddFinalizer(prog, internal.BpfmanOperatorFinalizer) + controllerutil.AddFinalizer(prog, finalizer) err := r.Update(ctx, prog) if err != nil { From fd95631d66186f1de4288dfe07ae33d9eec5b126 Mon Sep 17 00:00:00 2001 From: Mohamed Mahmoud Date: Tue, 11 Jun 2024 12:49:22 -0400 Subject: [PATCH 13/16] update samples config to include more ebpf program types Signed-off-by: Mohamed Mahmoud --- .../manifests/bpfman.io_bpfapplications.yaml | 6 ++- .../bpfman.io_v1alpha1_bpfapplication.yaml | 42 ++++++++++++------- 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/bundle/manifests/bpfman.io_bpfapplications.yaml b/bundle/manifests/bpfman.io_bpfapplications.yaml index 6053bd26f..b657004f2 100644 --- a/bundle/manifests/bpfman.io_bpfapplications.yaml +++ b/bundle/manifests/bpfman.io_bpfapplications.yaml @@ -14,7 +14,11 @@ spec: singular: bpfapplication scope: Cluster versions: - - name: v1alpha1 + - additionalPrinterColumns: + - jsonPath: .status.conditions[0].reason + name: Status + type: string + name: v1alpha1 schema: openAPIV3Schema: description: BpfApplication is the Schema for the bpfapplications API diff --git a/config/samples/bpfman.io_v1alpha1_bpfapplication.yaml b/config/samples/bpfman.io_v1alpha1_bpfapplication.yaml index 035f078d3..93d528891 100644 --- a/config/samples/bpfman.io_v1alpha1_bpfapplication.yaml +++ b/config/samples/bpfman.io_v1alpha1_bpfapplication.yaml @@ -9,26 +9,40 @@ spec: nodeselector: {} bytecode: image: - url: quay.io/bpfman-bytecode/testapp:latest - globaldata: - GLOBAL_u8: - - 0x01 - GLOBAL_u32: - - 0x0D - - 0x0C - - 0x0B - - 0x0A + url: quay.io/bpfman-bytecode/go-application-counter:latest programs: - - type: Fentry - fentry: - func_name: do_unlinkat - type: Kprobe kprobe: - bpffunctionname: my_kprobe + bpffunctionname: kprobe_counter func_name: try_to_wake_up offset: 0 retprobe: false - type: Tracepoint tracepoint: + bpffunctionname: tracepoint_kill_recorder names: - - syscalls/sys_enter_openat + - syscalls/sys_enter_kill + - type: TC + tc: + bpffunctionname: stats + interfaceselector: + primarynodeinterface: true + priority: 55 + direction: ingress + - type: Uprobe + uprobe: + bpffunctionname: uprobe_counter + func_name: main.getCount + target: /go-target + retprobe: false + containers: + namespace: go-target + pods: {} + containernames: + - go-target + - type: XDP + xdp: + bpffunctionname: xdp_stats + interfaceselector: + primarynodeinterface: true + priority: 55 \ No newline at end of file From 3f09cbfcc10ea9ab95c730cb279f63a878fecd88 Mon Sep 17 00:00:00 2001 From: Mohamed Mahmoud Date: Fri, 14 Jun 2024 09:05:16 -0400 Subject: [PATCH 14/16] address review comments Signed-off-by: Mohamed Mahmoud --- apis/v1alpha1/bpfapplication_types.go | 39 ++-- apis/v1alpha1/zz_generated.deepcopy.go | 5 + .../manifests/bpfman.io_bpfapplications.yaml | 170 ++++++++++++++++++ .../kustomization.yaml | 2 +- .../crd/bases/bpfman.io_bpfapplications.yaml | 170 ++++++++++++++++++ config/crd/kustomization.yaml | 2 +- .../bpfman.io_v1alpha1_bpfapplication.yaml | 13 +- .../bpfman-agent/application-program.go | 57 ------ .../bpfman-agent/application-program_test.go | 5 +- controllers/bpfman-agent/common.go | 5 +- internal/constants.go | 12 +- 11 files changed, 396 insertions(+), 84 deletions(-) diff --git a/apis/v1alpha1/bpfapplication_types.go b/apis/v1alpha1/bpfapplication_types.go index f6a8ad4dd..6b28064cd 100644 --- a/apis/v1alpha1/bpfapplication_types.go +++ b/apis/v1alpha1/bpfapplication_types.go @@ -24,44 +24,54 @@ import ( type EBPFProgType string const ( - // ProgTypeXDP refers to the eBPF XDP programs type. + // ProgTypeXDP refers to the XDP program type. ProgTypeXDP EBPFProgType = "XDP" - // ProgTypeTC refers to the eBPF TC programs type. + // ProgTypeTC refers to the TC program type. ProgTypeTC EBPFProgType = "TC" - // ProgTypeTCX refers to the eBPF TCx programs type. + // ProgTypeTCX refers to the TCx program type. ProgTypeTCX EBPFProgType = "TCX" - // ProgTypeFentry refers to the eBPF Fentry programs type. + // ProgTypeFentry refers to the Fentry program type. ProgTypeFentry EBPFProgType = "Fentry" - // ProgTypeFexit refers to the eBPF Fexit programs type. + // ProgTypeFexit refers to the Fexit program type. ProgTypeFexit EBPFProgType = "Fexit" - // ProgTypeKprobe refers to the eBPF Kprobe programs type. + // ProgTypeKprobe refers to the Kprobe program type. ProgTypeKprobe EBPFProgType = "Kprobe" - // ProgTypeKretprobe refers to the eBPF Kprobe programs type. + // ProgTypeKretprobe refers to the Kprobe program type. ProgTypeKretprobe EBPFProgType = "Kretprobe" - // ProgTypeUprobe refers to the eBPF Uprobe programs type. + // ProgTypeUprobe refers to the Uprobe program type. ProgTypeUprobe EBPFProgType = "Uprobe" - // ProgTypeUretprobe refers to the eBPF Uretprobe programs type. + // ProgTypeUretprobe refers to the Uretprobe program type. ProgTypeUretprobe EBPFProgType = "Uretprobe" - // ProgTypeTracepoint refers to the eBPF Tracepoint programs type. + // ProgTypeTracepoint refers to the Tracepoint program type. ProgTypeTracepoint EBPFProgType = "Tracepoint" ) // BpfApplicationProgram defines the desired state of BpfApplication +// +union +// +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'XDP' ? has(self.xdp) : !has(self.xdp)",message="xdp configuration is required when type is XDP, and forbidden otherwise" +// +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'TC' ? has(self.tc) : !has(self.tc)",message="tc configuration is required when type is TC, and forbidden otherwise" +// +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'TCX' ? has(self.tcx) : !has(self.tcx)",message="tcx configuration is required when type is TCX, and forbidden otherwise" +// +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'Fentry' ? has(self.fentry) : !has(self.fentry)",message="fentry configuration is required when type is Fentry, and forbidden otherwise" +// +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'Fexit' ? has(self.fexit) : !has(self.fexit)",message="fexit configuration is required when type is Fexit, and forbidden otherwise" +// +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'Kprobe' ? has(self.kprobe) : !has(self.kprobe)",message="kprobe configuration is required when type is Kprobe, and forbidden otherwise" +// +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'Kretprobe' ? has(self.kretprobe) : !has(self.kretprobe)",message="kretprobe configuration is required when type is Kretprobe, and forbidden otherwise" +// +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'Uprobe' ? has(self.uprobe) : !has(self.uprobe)",message="uprobe configuration is required when type is Uprobe, and forbidden otherwise" +// +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'Uretprobe' ? has(self.uretprobe) : !has(self.uretprobe)",message="uretprobe configuration is required when type is Uretprobe, and forbidden otherwise" +// +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'Tracepoint' ? has(self.tracepoint) : !has(self.tracepoint)",message="tracepoint configuration is required when type is Tracepoint, and forbidden otherwise" type BpfApplicationProgram struct { // Type specifies the bpf program type // +unionDiscriminator // +kubebuilder:validation:Required // +kubebuilder:validation:Enum:="XDP";"TC";"TCX";"Fentry";"Fexit";"Kprobe";"Kretprobe";"Uprobe";"Uretprobe";"Tracepoint" - // +optional Type EBPFProgType `json:"type,omitempty"` // xdp defines the desired state of the application's XdpPrograms. @@ -74,6 +84,11 @@ type BpfApplicationProgram struct { // +optional TC *TcProgramInfo `json:"tc,omitempty"` + // tcx defines the desired state of the application's TcPrograms. + // +unionMember + // +optional + TCX *TcProgramInfo `json:"tcx,omitempty"` + // fentry defines the desired state of the application's FentryPrograms. // +unionMember // +optional @@ -133,7 +148,9 @@ type BpfApplicationStatus struct { //+kubebuilder:resource:scope=Cluster // BpfApplication is the Schema for the bpfapplications API +// +kubebuilder:printcolumn:name="NodeSelector",type=string,JSONPath=`.spec.nodeselector` // +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.conditions[0].reason` +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" type BpfApplication struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/apis/v1alpha1/zz_generated.deepcopy.go b/apis/v1alpha1/zz_generated.deepcopy.go index ca038cf73..f2e712ba1 100644 --- a/apis/v1alpha1/zz_generated.deepcopy.go +++ b/apis/v1alpha1/zz_generated.deepcopy.go @@ -130,6 +130,11 @@ func (in *BpfApplicationProgram) DeepCopyInto(out *BpfApplicationProgram) { *out = new(TcProgramInfo) (*in).DeepCopyInto(*out) } + if in.TCX != nil { + in, out := &in.TCX, &out.TCX + *out = new(TcProgramInfo) + (*in).DeepCopyInto(*out) + } if in.Fentry != nil { in, out := &in.Fentry, &out.Fentry *out = new(FentryProgramInfo) diff --git a/bundle/manifests/bpfman.io_bpfapplications.yaml b/bundle/manifests/bpfman.io_bpfapplications.yaml index b657004f2..df54bbf6c 100644 --- a/bundle/manifests/bpfman.io_bpfapplications.yaml +++ b/bundle/manifests/bpfman.io_bpfapplications.yaml @@ -15,9 +15,15 @@ spec: scope: Cluster versions: - additionalPrinterColumns: + - jsonPath: .spec.nodeselector + name: NodeSelector + type: string - jsonPath: .status.conditions[0].reason name: Status type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date name: v1alpha1 schema: openAPIV3Schema: @@ -567,6 +573,129 @@ spec: - interfaceselector - priority type: object + tcx: + description: tcx defines the desired state of the application's + TcPrograms. + properties: + bpffunctionname: + description: |- + BpfFunctionName is the name of the function that is the entry point for the BPF + program + type: string + direction: + description: |- + Direction specifies the direction of traffic the tc program should + attach to for a given network device. + enum: + - ingress + - egress + type: string + interfaceselector: + description: Selector to determine the network interface + (or interfaces) + maxProperties: 1 + minProperties: 1 + properties: + interfaces: + description: |- + Interfaces refers to a list of network interfaces to attach the BPF + program to. + items: + type: string + type: array + primarynodeinterface: + description: Attach BPF program to the primary interface + on the node. Only 'true' accepted. + type: boolean + type: object + mapownerselector: + description: |- + MapOwnerSelector is used to select the loaded eBPF program this eBPF program + will share a map with. The value is a label applied to the BpfProgram to select. + The selector must resolve to exactly one instance of a BpfProgram on a given node + or the eBPF program will not load. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + priority: + description: |- + Priority specifies the priority of the tc program in relation to + other programs of the same type with the same attach point. It is a value + from 0 to 1000 where lower values have higher precedence. + format: int32 + maximum: 1000 + minimum: 0 + type: integer + proceedon: + default: + - pipe + - dispatcher_return + description: |- + ProceedOn allows the user to call other tc programs in chain on this exit code. + Multiple values are supported by repeating the parameter. + items: + enum: + - unspec + - ok + - reclassify + - shot + - pipe + - stolen + - queued + - repeat + - redirect + - trap + - dispatcher_return + type: string + maxItems: 11 + type: array + required: + - bpffunctionname + - direction + - interfaceselector + - priority + type: object tracepoint: description: tracepoint defines the desired state of the application's TracepointPrograms. @@ -1074,6 +1203,47 @@ spec: - priority type: object type: object + x-kubernetes-validations: + - message: xdp configuration is required when type is XDP, and forbidden + otherwise + rule: 'has(self.type) && self.type == ''XDP'' ? has(self.xdp) + : !has(self.xdp)' + - message: tc configuration is required when type is TC, and forbidden + otherwise + rule: 'has(self.type) && self.type == ''TC'' ? has(self.tc) : + !has(self.tc)' + - message: tcx configuration is required when type is TCX, and forbidden + otherwise + rule: 'has(self.type) && self.type == ''TCX'' ? has(self.tcx) + : !has(self.tcx)' + - message: fentry configuration is required when type is Fentry, + and forbidden otherwise + rule: 'has(self.type) && self.type == ''Fentry'' ? has(self.fentry) + : !has(self.fentry)' + - message: fexit configuration is required when type is Fexit, and + forbidden otherwise + rule: 'has(self.type) && self.type == ''Fexit'' ? has(self.fexit) + : !has(self.fexit)' + - message: kprobe configuration is required when type is Kprobe, + and forbidden otherwise + rule: 'has(self.type) && self.type == ''Kprobe'' ? has(self.kprobe) + : !has(self.kprobe)' + - message: kretprobe configuration is required when type is Kretprobe, + and forbidden otherwise + rule: 'has(self.type) && self.type == ''Kretprobe'' ? has(self.kretprobe) + : !has(self.kretprobe)' + - message: uprobe configuration is required when type is Uprobe, + and forbidden otherwise + rule: 'has(self.type) && self.type == ''Uprobe'' ? has(self.uprobe) + : !has(self.uprobe)' + - message: uretprobe configuration is required when type is Uretprobe, + and forbidden otherwise + rule: 'has(self.type) && self.type == ''Uretprobe'' ? has(self.uretprobe) + : !has(self.uretprobe)' + - message: tracepoint configuration is required when type is Tracepoint, + and forbidden otherwise + rule: 'has(self.type) && self.type == ''Tracepoint'' ? has(self.tracepoint) + : !has(self.tracepoint)' minItems: 1 type: array required: diff --git a/config/bpfman-operator-deployment/kustomization.yaml b/config/bpfman-operator-deployment/kustomization.yaml index 816ab6601..d0eb3dd30 100644 --- a/config/bpfman-operator-deployment/kustomization.yaml +++ b/config/bpfman-operator-deployment/kustomization.yaml @@ -5,4 +5,4 @@ kind: Kustomization images: - name: quay.io/bpfman/bpfman-operator newName: quay.io/bpfman/bpfman-operator - newTag: latest-amd64 + newTag: latest diff --git a/config/crd/bases/bpfman.io_bpfapplications.yaml b/config/crd/bases/bpfman.io_bpfapplications.yaml index 3e27e59d1..845d51f86 100644 --- a/config/crd/bases/bpfman.io_bpfapplications.yaml +++ b/config/crd/bases/bpfman.io_bpfapplications.yaml @@ -15,9 +15,15 @@ spec: scope: Cluster versions: - additionalPrinterColumns: + - jsonPath: .spec.nodeselector + name: NodeSelector + type: string - jsonPath: .status.conditions[0].reason name: Status type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date name: v1alpha1 schema: openAPIV3Schema: @@ -567,6 +573,129 @@ spec: - interfaceselector - priority type: object + tcx: + description: tcx defines the desired state of the application's + TcPrograms. + properties: + bpffunctionname: + description: |- + BpfFunctionName is the name of the function that is the entry point for the BPF + program + type: string + direction: + description: |- + Direction specifies the direction of traffic the tc program should + attach to for a given network device. + enum: + - ingress + - egress + type: string + interfaceselector: + description: Selector to determine the network interface + (or interfaces) + maxProperties: 1 + minProperties: 1 + properties: + interfaces: + description: |- + Interfaces refers to a list of network interfaces to attach the BPF + program to. + items: + type: string + type: array + primarynodeinterface: + description: Attach BPF program to the primary interface + on the node. Only 'true' accepted. + type: boolean + type: object + mapownerselector: + description: |- + MapOwnerSelector is used to select the loaded eBPF program this eBPF program + will share a map with. The value is a label applied to the BpfProgram to select. + The selector must resolve to exactly one instance of a BpfProgram on a given node + or the eBPF program will not load. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + priority: + description: |- + Priority specifies the priority of the tc program in relation to + other programs of the same type with the same attach point. It is a value + from 0 to 1000 where lower values have higher precedence. + format: int32 + maximum: 1000 + minimum: 0 + type: integer + proceedon: + default: + - pipe + - dispatcher_return + description: |- + ProceedOn allows the user to call other tc programs in chain on this exit code. + Multiple values are supported by repeating the parameter. + items: + enum: + - unspec + - ok + - reclassify + - shot + - pipe + - stolen + - queued + - repeat + - redirect + - trap + - dispatcher_return + type: string + maxItems: 11 + type: array + required: + - bpffunctionname + - direction + - interfaceselector + - priority + type: object tracepoint: description: tracepoint defines the desired state of the application's TracepointPrograms. @@ -1074,6 +1203,47 @@ spec: - priority type: object type: object + x-kubernetes-validations: + - message: xdp configuration is required when type is XDP, and forbidden + otherwise + rule: 'has(self.type) && self.type == ''XDP'' ? has(self.xdp) + : !has(self.xdp)' + - message: tc configuration is required when type is TC, and forbidden + otherwise + rule: 'has(self.type) && self.type == ''TC'' ? has(self.tc) : + !has(self.tc)' + - message: tcx configuration is required when type is TCX, and forbidden + otherwise + rule: 'has(self.type) && self.type == ''TCX'' ? has(self.tcx) + : !has(self.tcx)' + - message: fentry configuration is required when type is Fentry, + and forbidden otherwise + rule: 'has(self.type) && self.type == ''Fentry'' ? has(self.fentry) + : !has(self.fentry)' + - message: fexit configuration is required when type is Fexit, and + forbidden otherwise + rule: 'has(self.type) && self.type == ''Fexit'' ? has(self.fexit) + : !has(self.fexit)' + - message: kprobe configuration is required when type is Kprobe, + and forbidden otherwise + rule: 'has(self.type) && self.type == ''Kprobe'' ? has(self.kprobe) + : !has(self.kprobe)' + - message: kretprobe configuration is required when type is Kretprobe, + and forbidden otherwise + rule: 'has(self.type) && self.type == ''Kretprobe'' ? has(self.kretprobe) + : !has(self.kretprobe)' + - message: uprobe configuration is required when type is Uprobe, + and forbidden otherwise + rule: 'has(self.type) && self.type == ''Uprobe'' ? has(self.uprobe) + : !has(self.uprobe)' + - message: uretprobe configuration is required when type is Uretprobe, + and forbidden otherwise + rule: 'has(self.type) && self.type == ''Uretprobe'' ? has(self.uretprobe) + : !has(self.uretprobe)' + - message: tracepoint configuration is required when type is Tracepoint, + and forbidden otherwise + rule: 'has(self.type) && self.type == ''Tracepoint'' ? has(self.tracepoint) + : !has(self.tracepoint)' minItems: 1 type: array required: diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index ae82d432b..e70f0d3ea 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -36,7 +36,7 @@ patchesStrategicMerge: #- patches/cainjection_in_kprobeprograms.yaml #- patches/cainjection_in_uprobeprograms.yaml #- patches/cainjection_in_fentryprograms.yaml -#- patches/cainjection_in_fentryprograms.yaml +#- patches/cainjection_in_fexitrograms.yaml #- patches/cainjection_in_bpfapplications.yaml #+kubebuilder:scaffold:crdkustomizecainjectionpatch diff --git a/config/samples/bpfman.io_v1alpha1_bpfapplication.yaml b/config/samples/bpfman.io_v1alpha1_bpfapplication.yaml index 93d528891..18e1a0c9e 100644 --- a/config/samples/bpfman.io_v1alpha1_bpfapplication.yaml +++ b/config/samples/bpfman.io_v1alpha1_bpfapplication.yaml @@ -32,14 +32,17 @@ spec: - type: Uprobe uprobe: bpffunctionname: uprobe_counter - func_name: main.getCount - target: /go-target + func_name: malloc + target: libc retprobe: false containers: - namespace: go-target - pods: {} + namespace: bpfman + pods: + matchLabels: + name: bpfman-daemon containernames: - - go-target + - bpfman + - bpfman-agent - type: XDP xdp: bpffunctionname: xdp_stats diff --git a/controllers/bpfman-agent/application-program.go b/controllers/bpfman-agent/application-program.go index b5e94a6fe..554eef07c 100644 --- a/controllers/bpfman-agent/application-program.go +++ b/controllers/bpfman-agent/application-program.go @@ -26,10 +26,6 @@ type BpfApplicationReconciler struct { ourNode *v1.Node } -// TODO: As implemented, the BpfApplicationReconciler doesn't need to implement -// the bpfmanReconciler interface. We should think about what's needed and what -// isn't. - func (r *BpfApplicationReconciler) getRecType() string { return internal.ApplicationString } @@ -93,8 +89,6 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque // Reconcile FentryProgram. complete, res, err = r.reconcileCommon(ctx, rec, fentryObjects) - r.showPrograms(ctx, rec) - case bpfmaniov1alpha1.ProgTypeFexit: fexitProgram := bpfmaniov1alpha1.FexitProgram{ ObjectMeta: metav1.ObjectMeta{ @@ -115,8 +109,6 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque // Reconcile FexitProgram. complete, res, err = r.reconcileCommon(ctx, rec, fexitObjects) - r.showPrograms(ctx, rec) - case bpfmaniov1alpha1.ProgTypeKprobe, bpfmaniov1alpha1.ProgTypeKretprobe: kprobeProgram := bpfmaniov1alpha1.KprobeProgram{ @@ -138,8 +130,6 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque // Reconcile KprobeProgram or KpretprobeProgram. complete, res, err = r.reconcileCommon(ctx, rec, kprobeObjects) - r.showPrograms(ctx, rec) - case bpfmaniov1alpha1.ProgTypeUprobe, bpfmaniov1alpha1.ProgTypeUretprobe: uprobeProgram := bpfmaniov1alpha1.UprobeProgram{ @@ -161,8 +151,6 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque // Reconcile UprobeProgram or UpretprobeProgram. complete, res, err = r.reconcileCommon(ctx, rec, uprobeObjects) - r.showPrograms(ctx, rec) - case bpfmaniov1alpha1.ProgTypeTracepoint: tracepointProgram := bpfmaniov1alpha1.TracepointProgram{ ObjectMeta: metav1.ObjectMeta{ @@ -183,8 +171,6 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque // Reconcile TracepointProgram. complete, res, err = r.reconcileCommon(ctx, rec, tracepointObjects) - r.showPrograms(ctx, rec) - case bpfmaniov1alpha1.ProgTypeTC, bpfmaniov1alpha1.ProgTypeTCX: tcProgram := bpfmaniov1alpha1.TcProgram{ @@ -206,8 +192,6 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque // Reconcile TcProgram. complete, res, err = r.reconcileCommon(ctx, rec, tcObjects) - r.showPrograms(ctx, rec) - case bpfmaniov1alpha1.ProgTypeXDP: xdpProgram := bpfmaniov1alpha1.XdpProgram{ ObjectMeta: metav1.ObjectMeta{ @@ -228,10 +212,6 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque // Reconcile XdpProgram. complete, res, err = r.reconcileCommon(ctx, rec, xdpObjects) - r.showPrograms(ctx, rec) - - r.showPrograms(ctx, rec) - default: r.Logger.Error(fmt.Errorf("unsupported bpf program type"), "unsupported bpf program type", "ProgType", p.Type) // Skip this program and continue to the next one @@ -260,43 +240,6 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque return res, err } -// TODO: Remove this debug function -func (r *BpfApplicationReconciler) getAllExistingBpfPrograms(ctx context.Context, - rec bpfmanReconciler) (map[string]bpfmaniov1alpha1.BpfProgram, error) { - - bpfProgramList := &bpfmaniov1alpha1.BpfProgramList{} - - // Only list bpfPrograms for this *Program and the controller's node - opts := []client.ListOption{ - client.MatchingLabels{ - internal.BpfProgramOwnerLabel: rec.getOwner().GetName(), - internal.K8sHostLabel: r.NodeName, - }, - } - - err := r.List(ctx, bpfProgramList, opts...) - if err != nil { - return nil, err - } - - existingBpfPrograms := map[string]bpfmaniov1alpha1.BpfProgram{} - for _, bpfProg := range bpfProgramList.Items { - existingBpfPrograms[bpfProg.GetName()] = bpfProg - } - - return existingBpfPrograms, nil -} - -// TODO: Remove this debug function -func (r *BpfApplicationReconciler) showPrograms(ctx context.Context, rec bpfmanReconciler) { - programs, err := r.getAllExistingBpfPrograms(ctx, rec) - if err != nil { - r.Logger.V(1).Info("Failed to get existing BpfPrograms", "Application", rec.getOwner().GetName(), "Error", err) - } else { - r.Logger.V(1).Info("Existing BpfPrograms", "Application", rec.getOwner().GetName(), "Programs", programs) - } -} - // SetupWithManager sets up the controller with the Manager. // The Bpfman-Agent should reconcile whenever a BpfApplication object is updated, // load the programs to the node via bpfman, and then create a bpfProgram object diff --git a/controllers/bpfman-agent/application-program_test.go b/controllers/bpfman-agent/application-program_test.go index bdd61a4f9..f6d668d10 100644 --- a/controllers/bpfman-agent/application-program_test.go +++ b/controllers/bpfman-agent/application-program_test.go @@ -97,8 +97,6 @@ func TestBpfApplicationControllerCreate(t *testing.T) { s.AddKnownTypes(bpfmaniov1alpha1.SchemeGroupVersion, &bpfmaniov1alpha1.BpfApplication{}) s.AddKnownTypes(bpfmaniov1alpha1.SchemeGroupVersion, &bpfmaniov1alpha1.BpfProgramList{}) s.AddKnownTypes(bpfmaniov1alpha1.SchemeGroupVersion, &bpfmaniov1alpha1.BpfProgram{}) - s.AddKnownTypes(bpfmaniov1alpha1.SchemeGroupVersion, &bpfmaniov1alpha1.FentryProgramList{}) - s.AddKnownTypes(bpfmaniov1alpha1.SchemeGroupVersion, &bpfmaniov1alpha1.KprobeProgramList{}) // Create a fake client to mock API calls. cl := fake.NewClientBuilder().WithStatusSubresource(App).WithStatusSubresource(&bpfmaniov1alpha1.BpfProgram{}).WithRuntimeObjects(objs...).Build() @@ -128,7 +126,6 @@ func TestBpfApplicationControllerCreate(t *testing.T) { }, } - // do fentry program // First reconcile should create the bpf program object res, err := r.Reconcile(ctx, req) if err != nil { @@ -168,7 +165,7 @@ func TestBpfApplicationControllerCreate(t *testing.T) { // Require no requeue require.False(t, res.Requeue) - // 1- do Fentry Program + // do Fentry Program expectedLoadReq := &gobpfman.LoadRequest{ Bytecode: &gobpfman.BytecodeLocation{ Location: &gobpfman.BytecodeLocation_File{File: bytecodePath}, diff --git a/controllers/bpfman-agent/common.go b/controllers/bpfman-agent/common.go index 91e592f80..ba2672ae5 100644 --- a/controllers/bpfman-agent/common.go +++ b/controllers/bpfman-agent/common.go @@ -128,7 +128,7 @@ type bpfmanReconciler interface { // reconcileCommon is the common reconciler loop called by each bpfman // reconciler. It reconciles each program in the list. The boolean return // value is set to true if we've made it through all the programs in the list -// without anything being updated and a reque has not been requested. Otherwise, +// without anything being updated and a requeue has not been requested. Otherwise, // it's set to false. reconcileCommon should not return error because it will // trigger an infinite reconcile loop. Instead, it should report the error to // user and retry if specified. For some errors the controller may decide not to @@ -222,7 +222,6 @@ func (r *ReconcilerCommon) reconcileBpfProgram(ctx context.Context, isSame, reasons := bpfmanagentinternal.DoesProgExist(loadedBpfProgram, loadRequest) if !isSame { r.Logger.V(1).Info("bpf program is in wrong state, unloading and reloading", "reason", reasons, "bpfProgram Name", bpfProgram.Name, "bpf program ID", id) - r.Logger.V(1).WithValues("loadRequest", loadRequest).WithValues("loadedBpfProgram", loadedBpfProgram).Info("bpf program state") if err := bpfmanagentinternal.UnloadBpfmanProgram(ctx, r.BpfmanClient, *id); err != nil { r.Logger.Error(err, "Failed to unload BPF Program") return bpfmaniov1alpha1.BpfProgCondNotUnloaded, nil @@ -533,7 +532,7 @@ func (r *ReconcilerCommon) createBpfProgram( annotations map[string]string) (*bpfmaniov1alpha1.BpfProgram, error) { r.Logger.V(1).Info("createBpfProgram()", "Name", bpfProgramName, - "Owner", rec.getOwner().GetName(), "OwnerType", rec.getRecType()) + "Owner", rec.getOwner().GetName(), "OwnerType", rec.getRecType(), "Name", rec.getName()) bpfProg := &bpfmaniov1alpha1.BpfProgram{ ObjectMeta: metav1.ObjectMeta{ diff --git a/internal/constants.go b/internal/constants.go index 764ff2cd4..8ddc43a06 100644 --- a/internal/constants.go +++ b/internal/constants.go @@ -28,8 +28,6 @@ const ( UprobeNoContainersOnNode = "bpfman.io.uprobeprogramcontroller/nocontainersonnode" FentryProgramFunction = "bpfman.io.fentryprogramcontroller/function" FexitProgramFunction = "bpfman.io.fexitprogramcontroller/function" - BpfProgramOwnerLabel = "bpfman.io/ownedByProgram" - BpfParentProgram = "bpfman.io/parentProgram" K8sHostLabel = "kubernetes.io/hostname" DiscoveredLabel = "bpfman.io/discoveredProgram" IdAnnotation = "bpfman.io/ProgramId" @@ -51,6 +49,16 @@ const ( DefaultPath = "/run/bpfman-sock/bpfman.sock" DefaultPort = 50051 DefaultEnabled = true + // BpfProgramOwnerLabel is the name of the object that owns the BpfProgram + // object. In the case of a *Program, it will be the name of the *Program + // object. In the case of a BpfApplication, it will be the name of the + // BpfApplication object. + BpfProgramOwnerLabel = "bpfman.io/ownedByProgram" + // BpfParentProgram is the name of the current program that caused the + // creation of the BpfProgram object. In the case of a *Program, it will be + // the name of the *Program object. In the case of a BpfApplication, it + // will be the name generated for the given BpfApplication program. + BpfParentProgram = "bpfman.io/parentProgram" ) // ----------------------------------------------------------------------------- From b5c665d6ce4ab62f15d23ed03cf4f1bb7697696f Mon Sep 17 00:00:00 2001 From: Mohamed Mahmoud Date: Sat, 15 Jun 2024 09:53:50 -0400 Subject: [PATCH 15/16] update readme with the new bpfapplication object Signed-off-by: Mohamed Mahmoud --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 3dde146a2..01632884c 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,12 @@ bpfman supports: * `uprobeProgram` * `xdpProgram` +## BpfApplication CRD + +The `BpfApplication` CRD is designed for managing eBPF programs at an application level within a Kubernetes cluster. +This CRD allows Kubernetes users to define which eBPF programs are essential for an application's operations and specify +how these programs should be deployed across the cluster. + ## BpfProgram CRD The `BpfProgram` CRD is used internally by the `bpfman-deployment` to keep track of per-node `bpfman` state, From 5fc5db8d416d1e8bcd498578493bb9fbfbd1a341 Mon Sep 17 00:00:00 2001 From: Andre Fredette Date: Tue, 18 Jun 2024 18:09:20 -0400 Subject: [PATCH 16/16] Make BpfApplication BpfProgram names unique Adds the attach point to the name of the BpfApplication programs so that BpfPrograms created by multipl programs of the same type are unique. Also - Adds some helper functions for name generation - Fixed an indent issue with the BpfApplication sample yaml Signed-off-by: Andre Fredette --- ...bpfman-operator.clusterserviceversion.yaml | 1779 ++++++++++------- .../bpfman.io_v1alpha1_bpfapplication.yaml | 6 +- .../bpfman-agent/application-program.go | 33 +- .../bpfman-agent/application-program_test.go | 4 +- controllers/bpfman-agent/common.go | 7 + controllers/bpfman-agent/fentry-program.go | 4 +- controllers/bpfman-agent/fexit-program.go | 4 +- controllers/bpfman-agent/kprobe-program.go | 4 +- .../bpfman-agent/tracepoint-program.go | 4 +- controllers/bpfman-agent/uprobe-program.go | 4 +- 10 files changed, 1112 insertions(+), 737 deletions(-) diff --git a/bundle/manifests/bpfman-operator.clusterserviceversion.yaml b/bundle/manifests/bpfman-operator.clusterserviceversion.yaml index 80761a387..28b29f3a0 100644 --- a/bundle/manifests/bpfman-operator.clusterserviceversion.yaml +++ b/bundle/manifests/bpfman-operator.clusterserviceversion.yaml @@ -2,11 +2,301 @@ apiVersion: operators.coreos.com/v1alpha1 kind: ClusterServiceVersion metadata: annotations: - alm-examples: "[]" + alm-examples: |- + [ + { + "apiVersion": "bpfman.io/v1alpha1", + "kind": "BpfApplication", + "metadata": { + "labels": { + "app.kubernetes.io/name": "bpfapplication" + }, + "name": "bpfapplication-sample" + }, + "spec": { + "bytecode": { + "image": { + "url": "quay.io/bpfman-bytecode/go-application-counter:latest" + } + }, + "nodeselector": {}, + "programs": [ + { + "kprobe": { + "bpffunctionname": "kprobe_counter", + "func_name": "try_to_wake_up", + "offset": 0, + "retprobe": false + }, + "type": "Kprobe" + }, + { + "tracepoint": { + "bpffunctionname": "tracepoint_kill_recorder", + "names": [ + "syscalls/sys_enter_kill" + ] + }, + "type": "Tracepoint" + }, + { + "tc": { + "bpffunctionname": "stats", + "direction": "ingress", + "interfaceselector": { + "primarynodeinterface": true + }, + "priority": 55 + }, + "type": "TC" + }, + { + "type": "Uprobe", + "uprobe": { + "bpffunctionname": "uprobe_counter", + "containers": { + "containernames": [ + "bpfman", + "bpfman-agent" + ], + "namespace": "bpfman", + "pods": { + "matchLabels": { + "name": "bpfman-daemon" + } + } + }, + "func_name": "malloc", + "retprobe": false, + "target": "libc" + } + }, + { + "type": "XDP", + "xdp": { + "bpffunctionname": "xdp_stats", + "interfaceselector": { + "primarynodeinterface": true + }, + "priority": 55 + } + } + ] + } + }, + { + "apiVersion": "bpfman.io/v1alpha1", + "kind": "FentryProgram", + "metadata": { + "labels": { + "app.kubernetes.io/name": "fentryprogram" + }, + "name": "fentry-example" + }, + "spec": { + "bpffunctionname": "test_fentry", + "bytecode": { + "image": { + "url": "quay.io/bpfman-bytecode/fentry:latest" + } + }, + "func_name": "do_unlinkat", + "nodeselector": {} + } + }, + { + "apiVersion": "bpfman.io/v1alpha1", + "kind": "FexitProgram", + "metadata": { + "labels": { + "app.kubernetes.io/name": "fexitprogram" + }, + "name": "fexit-example" + }, + "spec": { + "bpffunctionname": "test_fexit", + "bytecode": { + "image": { + "url": "quay.io/bpfman-bytecode/fexit:latest" + } + }, + "func_name": "do_unlinkat", + "nodeselector": {} + } + }, + { + "apiVersion": "bpfman.io/v1alpha1", + "kind": "KprobeProgram", + "metadata": { + "labels": { + "app.kubernetes.io/name": "kprobeprogram" + }, + "name": "kprobe-example" + }, + "spec": { + "bpffunctionname": "my_kprobe", + "bytecode": { + "image": { + "url": "quay.io/bpfman-bytecode/kprobe:latest" + } + }, + "func_name": "try_to_wake_up", + "globaldata": { + "GLOBAL_u32": [ + 13, + 12, + 11, + 10 + ], + "GLOBAL_u8": [ + 1 + ] + }, + "nodeselector": {}, + "offset": 0, + "retprobe": false + } + }, + { + "apiVersion": "bpfman.io/v1alpha1", + "kind": "TcProgram", + "metadata": { + "labels": { + "app.kubernetes.io/name": "tcprogram" + }, + "name": "tc-pass-all-nodes" + }, + "spec": { + "bpffunctionname": "pass", + "bytecode": { + "image": { + "url": "quay.io/bpfman-bytecode/tc_pass:latest" + } + }, + "direction": "ingress", + "globaldata": { + "GLOBAL_u32": [ + 13, + 12, + 11, + 10 + ], + "GLOBAL_u8": [ + 1 + ] + }, + "interfaceselector": { + "primarynodeinterface": true + }, + "nodeselector": {}, + "priority": 0 + } + }, + { + "apiVersion": "bpfman.io/v1alpha1", + "kind": "TracepointProgram", + "metadata": { + "labels": { + "app.kubernetes.io/name": "tracepointprogram" + }, + "name": "tracepoint-example" + }, + "spec": { + "bpffunctionname": "enter_openat", + "bytecode": { + "image": { + "url": "quay.io/bpfman-bytecode/tracepoint:latest" + } + }, + "globaldata": { + "GLOBAL_u32": [ + 13, + 12, + 11, + 10 + ], + "GLOBAL_u8": [ + 1 + ] + }, + "names": [ + "syscalls/sys_enter_openat" + ], + "nodeselector": {} + } + }, + { + "apiVersion": "bpfman.io/v1alpha1", + "kind": "UprobeProgram", + "metadata": { + "labels": { + "app.kubernetes.io/name": "uprobeprogram" + }, + "name": "uprobe-example" + }, + "spec": { + "bpffunctionname": "my_uprobe", + "bytecode": { + "image": { + "url": "quay.io/bpfman-bytecode/uprobe:latest" + } + }, + "func_name": "syscall", + "globaldata": { + "GLOBAL_u32": [ + 13, + 12, + 11, + 10 + ], + "GLOBAL_u8": [ + 1 + ] + }, + "nodeselector": {}, + "retprobe": false, + "target": "libc" + } + }, + { + "apiVersion": "bpfman.io/v1alpha1", + "kind": "XdpProgram", + "metadata": { + "labels": { + "app.kubernetes.io/name": "xdpprogram" + }, + "name": "xdp-pass-all-nodes" + }, + "spec": { + "bpffunctionname": "pass", + "bytecode": { + "image": { + "url": "quay.io/bpfman-bytecode/xdp_pass:latest" + } + }, + "globaldata": { + "GLOBAL_u32": [ + 13, + 12, + 11, + 10 + ], + "GLOBAL_u8": [ + 1 + ] + }, + "interfaceselector": { + "primarynodeinterface": true + }, + "nodeselector": {}, + "priority": 0 + } + } + ] capabilities: Basic Install categories: OpenShift Optional containerImage: quay.io/bpfman/bpfman-operator:v0.0.0 - createdAt: "2024-06-07T18:48:53Z" + createdAt: "2024-06-20T14:00:22Z" operatorframework.io/suggested-namespace-template: |- { "apiVersion": "v1", @@ -30,9 +320,52 @@ metadata: namespace: placeholder spec: apiservicedefinitions: {} - customresourcedefinitions: {} - description: - "The bpfman Operator is a Kubernetes Operator for deploying [bpfman](https://bpfman.netlify.app/), + customresourcedefinitions: + owned: + - kind: BpfApplication + name: bpfapplications.bpfman.io + version: v1alpha1 + - description: BpfProgram is the Schema for the BpfProgram API + displayName: Bpf Program + kind: BpfProgram + name: bpfprograms.bpfman.io + version: v1alpha1 + - description: FentryProgram is the Schema for the Fentryprograms API + displayName: Fentry Program + kind: FentryProgram + name: fentryprograms.bpfman.io + version: v1alpha1 + - description: FexitProgram is the Schema for the Fexitprograms API + displayName: Fexit Program + kind: FexitProgram + name: fexitprograms.bpfman.io + version: v1alpha1 + - description: KprobeProgram is the Schema for the Kprobeprograms API + displayName: Kprobe Program + kind: KprobeProgram + name: kprobeprograms.bpfman.io + version: v1alpha1 + - description: TcProgram is the Schema for the Tcprograms API + displayName: Tc Program + kind: TcProgram + name: tcprograms.bpfman.io + version: v1alpha1 + - description: TracepointProgram is the Schema for the Tracepointprograms API + displayName: Tracepoint Program + kind: TracepointProgram + name: tracepointprograms.bpfman.io + version: v1alpha1 + - description: UprobeProgram is the Schema for the Uprobeprograms API + displayName: Uprobe Program + kind: UprobeProgram + name: uprobeprograms.bpfman.io + version: v1alpha1 + - description: XdpProgram is the Schema for the Xdpprograms API + displayName: Xdp Program + kind: XdpProgram + name: xdpprograms.bpfman.io + version: v1alpha1 + description: "The bpfman Operator is a Kubernetes Operator for deploying [bpfman](https://bpfman.netlify.app/), a system daemon\nfor managing eBPF programs. It deploys bpfman itself along with CRDs to make deploying\neBPF programs in Kubernetes much easier.\n\n## Quick Start\n\nTo get bpfman up and running quickly simply click 'install' to deploy the bpfman-operator @@ -51,723 +384,749 @@ spec: checkout the [bpfman community website](https://bpfman.io/) for more information." displayName: Bpfman Operator icon: - - base64data: | - PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+Cjwh - RE9DVFlQRSBzdmcgUFVCTElDICItLy9XM0MvL0RURCBTVkcgMS4xLy9FTiIgImh0dHA6Ly93d3cu - dzMub3JnL0dyYXBoaWNzL1NWRy8xLjEvRFREL3N2ZzExLmR0ZCI+CjwhLS0gQ3JlYXRlZCB3aXRo - IFZlY3Rvcm5hdG9yIChodHRwOi8vdmVjdG9ybmF0b3IuaW8vKSAtLT4KPHN2ZyBoZWlnaHQ9IjEw - MCUiIHN0cm9rZS1taXRlcmxpbWl0PSIxMCIgc3R5bGU9ImZpbGwtcnVsZTpub256ZXJvO2NsaXAt - cnVsZTpldmVub2RkO3N0cm9rZS1saW5lY2FwOnJvdW5kO3N0cm9rZS1saW5lam9pbjpyb3VuZDsi - IHZlcnNpb249IjEuMSIgdmlld0JveD0iMCAwIDEwMjQgMTAyNCIgd2lkdGg9IjEwMCUiIHhtbDpz - cGFjZT0icHJlc2VydmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgeG1sbnM6 - dmVjdG9ybmF0b3I9Imh0dHA6Ly92ZWN0b3JuYXRvci5pbyIgeG1sbnM6eGxpbms9Imh0dHA6Ly93 - d3cudzMub3JnLzE5OTkveGxpbmsiPgo8ZGVmcz4KPGxpbmVhckdyYWRpZW50IGdyYWRpZW50VHJh - bnNmb3JtPSJtYXRyaXgoMS40NTY4MyAxLjQ1NjgzIC0xLjQ1NjgzIDEuNDU2ODMgNzkxLjI0NCAt - NzA0LjEzMykiIGdyYWRpZW50VW5pdHM9InVzZXJTcGFjZU9uVXNlIiBpZD0iTGluZWFyR3JhZGll - bnQiIHgxPSIzNjguNDYyIiB4Mj0iMjQyLjMzMSIgeTE9IjQyNy4wNTMiIHkyPSI1NTMuNjE0Ij4K - PHN0b3Agb2Zmc2V0PSIwLjQyMjU5MiIgc3RvcC1jb2xvcj0iIzMzMzEyYyIvPgo8c3RvcCBvZmZz - ZXQ9IjEiIHN0b3AtY29sb3I9IiNmM2M2MjIiLz4KPC9saW5lYXJHcmFkaWVudD4KPHBhdGggZD0i - TTQzNS4xMTMgNDQ4LjQxM0M0MzUuMTEzIDM3NS43MjMgNDkzLjc5MyAyNjguNjkyIDUxMS4yMDkg - MjY4LjY5MkM1MjguNjI1IDI2OC42OTIgNTg4Ljg3MyAzNzAuOTU3IDU4OC44NzMgNDQzLjY0NkM1 - ODguODczIDUxNi4zMzYgNTMyLjc4NSA2MDAuNzc3IDUxMS4yMDkgNjAwLjc3N0M0ODkuNjMzIDYw - MC43NzcgNDM1LjExMyA1MjEuMTAzIDQzNS4xMTMgNDQ4LjQxM1oiIGlkPSJGaWxsIi8+CjxwYXRo - IGQ9Ik00MzUuMTEzIDQ0OC40MTNDNDM1LjExMyAzNzUuNzIzIDQ5My43OTMgMjY4LjY5MiA1MTEu - MjA5IDI2OC42OTJDNTI4LjYyNSAyNjguNjkyIDU4OC44NzMgMzcwLjk1NyA1ODguODczIDQ0My42 - NDZDNTg4Ljg3MyA1MTYuMzM2IDUzMi43ODUgNjAwLjc3NyA1MTEuMjA5IDYwMC43NzdDNDg5LjYz - MyA2MDAuNzc3IDQzNS4xMTMgNTIxLjEwMyA0MzUuMTEzIDQ0OC40MTNaIiBpZD0iRmlsbF8yIi8+ - CjxwYXRoIGQ9Ik00MzUuMTEzIDQ0OC40MTNDNDM1LjExMyAzNzUuNzIzIDQ5My43OTMgMjY4LjY5 - MiA1MTEuMjA5IDI2OC42OTJDNTI4LjYyNSAyNjguNjkyIDU4OC44NzMgMzcwLjk1NyA1ODguODcz - IDQ0My42NDZDNTg4Ljg3MyA1MTYuMzM2IDUzMi43ODUgNjAwLjc3NyA1MTEuMjA5IDYwMC43NzdD - NDg5LjYzMyA2MDAuNzc3IDQzNS4xMTMgNTIxLjEwMyA0MzUuMTEzIDQ0OC40MTNaIiBpZD0iRmls - bF8zIi8+CjxwYXRoIGQ9Ik00MzUuMTIgMzY4LjgzOEM0MzUuMTIgMzY4LjgzOCA0NzQuMjE5IDM3 - OC4yNjMgNTEyLjAwNyAzNzguMzM1QzU0OS43OTYgMzc4LjQwNyA1ODguODggMzY5LjM4NSA1ODgu - ODggMzY5LjM4NSIgaWQ9IkZpbGxfNCIvPgo8L2RlZnM+CjxnIGlkPSJMYXllci0xIiB2ZWN0b3Ju - YXRvcjpsYXllck5hbWU9IkxheWVyIDEiPgo8ZyBvcGFjaXR5PSIxIiB2ZWN0b3JuYXRvcjpsYXll - ck5hbWU9Ikdyb3VwIDEyIj4KPHBhdGggZD0iTTE4Ni4yMzMgMzg3LjI3OEMxODYuMjMzIDIwNy4z - NjQgMzMyLjA4NCA2MS41MTM5IDUxMS45OTggNjEuNTEzOUM2OTEuOTE2IDYxLjUxMzkgODM3Ljc2 - NyAyMDcuMzY0IDgzNy43NjcgMzg3LjI3OEM4MzcuNzY3IDU2Ny4xOTMgNjkxLjkxNiA3MTMuMDQz - IDUxMS45OTggNzEzLjA0M0MzMzIuMDg0IDcxMy4wNDMgMTg2LjIzMyA1NjcuMTkzIDE4Ni4yMzMg - Mzg3LjI3OCIgZmlsbD0iI2YyOGYyMiIgZmlsbC1ydWxlPSJub256ZXJvIiBvcGFjaXR5PSIxIiBz - dHJva2U9Im5vbmUiIHZlY3Rvcm5hdG9yOmxheWVyTmFtZT0icGF0aCIvPgo8ZyBvcGFjaXR5PSIx - IiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9Ikdyb3VwIDEzIj4KPHBhdGggZD0iTTIwMS4zMzEgNDg1 - LjQ4NUMyMDQuODQ3IDQ4MS41MzQgMjA4LjE4MiA0NzcuNTkzIDIxMS4yOTQgNDczLjY2QzI3MS4z - ODUgMzk3LjYzNiAyNDkuMTUgMzQzLjc2OSAxOTguMTU5IDI5OS45OTFDMTkwLjQ0MyAzMjcuNzg0 - IDE4Ni4yMzUgMzU3LjAzIDE4Ni4yMzUgMzg3LjI3N0MxODYuMjM1IDQyMS41MDkgMTkxLjU0NCA0 - NTQuNDkgMjAxLjMzMSA0ODUuNDg1IiBmaWxsPSIjZTBmMjIyIiBmaWxsLXJ1bGU9Im5vbnplcm8i - IG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9uZSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJwYXRoIi8+ - CjxwYXRoIGQ9Ik00MjEuMTYzIDQwNS4wOThDNDQzLjY2IDMyMy4yNzkgMzQxLjE0MSAyNTIuNjQx - IDI0Ni4xNzkgMTk5LjA4QzIzOSAyMDkuMTk4IDIzMi4zNDIgMjE5LjcwMyAyMjYuMzM4IDIzMC42 - MjlDMzA5LjI2NSAyNzguMTY5IDM5MC43NjkgMzQyLjM1OSAzNTYuNjU0IDQyMy4zMDFDMzM0LjIy - NSA0NzYuNTA1IDI3OC43MTMgNTEzLjMwNyAyMzIuNzA4IDU1NS4wMDFDMjM4LjYxNCA1NjQuODE4 - IDI0NS4wMjcgNTc0LjI5MiAyNTEuOTA0IDU4My4zOTZDMzA5Ljk0NCA1MjQuODAxIDM5OS4zNTMg - NDg0LjQyNyA0MjEuMTYzIDQwNS4wOTgiIGZpbGw9IiNlMGYyMjIiIGZpbGwtcnVsZT0ibm9uemVy - byIgb3BhY2l0eT0iMSIgc3Ryb2tlPSJub25lIiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9InBhdGgi - Lz4KPHBhdGggZD0iTTU5Ny45MjMgMzIzLjgwN0M2MjkuMjkyIDQ0Mi45OTkgNDU0LjQwMiA1MTku - MzQxIDM4OC45NiA1OTIuMzA1QzM2Ni4zMDQgNjE3LjU2NiAzNDguODAxIDYzOS4xODcgMzM5LjMw - NiA2NjMuNDY0QzM0OC43NTggNjY5LjM4NyAzNTguNTE1IDY3NC44NTggMzY4LjU4NiA2NzkuODAx - QzM3My43NzQgNjU2LjU2NSAzODQuNTUgNjMyLjg1MSA0MDIuODQgNjA4LjIzNUM0NzEuODAyIDUx - NS40MTcgNjUwLjg2NSA0NTUuNTg1IDYyNi4zMTMgMzE4LjkwM0M2MDcuNzE4IDIxNS4zODYgNDk4 - LjE3NiAxNjEuODQyIDQ2MS45MjQgNjUuMzQ5N0M0MzguOTk3IDY4Ljg4NzIgNDE2Ljg5NSA3NC44 - NzQ4IDM5NS44MDggODIuOTI5OEM0NDcuMzAxIDE3My44MTMgNTcwLjgyNiAyMjAuODQxIDU5Ny45 - MjMgMzIzLjgwNyIgZmlsbD0iI2UwZjIyMiIgZmlsbC1ydWxlPSJub256ZXJvIiBvcGFjaXR5PSIx - IiBzdHJva2U9Im5vbmUiIHZlY3Rvcm5hdG9yOmxheWVyTmFtZT0icGF0aCIvPgo8cGF0aCBkPSJN - NzA1LjQ4MSAzMDQuMzA1Qzc0MC4zNzkgNDYwLjIwOSA1MTkuNjMxIDUyMy4yNjUgNDQ2LjAyNiA2 - MzAuMzQ1QzQzMC40ODQgNjUyLjk0OSA0MjMuMDc4IDY3Ni4zNzEgNDIxLjI1OSA3MDAuMTQxQzQz - NC4zMjYgNzAzLjkyMyA0NDcuNjk4IDcwNi45NzUgNDYxLjM4NCA3MDkuMTExQzQ2Mi43NzIgNjg0 - LjQ2OSA0NjkuOCA2NjAuMjI2IDQ4NS4xMTUgNjM2Ljg4N0M1NTguODAxIDUyNC41ODUgNzg2Ljk3 - MiA0NjguMDg4IDc2MC4xMDggMzA4LjUxM0M3NDcuMzg1IDIzMi45MjcgNzAwLjQ0MyAxNzQuMzU0 - IDY3NS4zNDEgMTA1LjQ2NUM2NTAuMDI4IDkwLjc2MDggNjIyLjU5NiA3OS4yOTMgNTkzLjU0OSA3 - MS44MDUzQzYxOC4xODYgMTU1Ljg1OSA2ODUuNzY0IDIxNi4yMzcgNzA1LjQ4MSAzMDQuMzA1IiBm - aWxsPSIjZTBmMjIyIiBmaWxsLXJ1bGU9Im5vbnplcm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9u - ZSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJwYXRoIi8+CjxwYXRoIGQ9Ik0zMjQuMzM5IDYxMS4y - MjNDMzgxLjUzNyA1MzUuNzk2IDU2My4wOCA0NjMuODc3IDU1Mi43MyAzNTIuNzQ4QzU0My41OTIg - MjU0LjY0OCA0MDIuNjYzIDE5NC4wNzYgMzE2LjE1NSAxMjYuOTYyQzMwNi40NDEgMTM0LjI4MiAy - OTcuMiAxNDIuMTc0IDI4OC4zNzUgMTUwLjUxM0MzODUuNjg1IDIxMS44NTQgNTE1Ljk3MSAyNzcu - NDc2IDUxMy40MzUgMzY1LjIzOUM1MTAuMDk5IDQ4MC42MzIgMzQ3LjM3NCA1MzMuNDMyIDI4NC44 - ODEgNjIwLjcxM0MyOTEuNzU0IDYyNy40MDIgMjk4Ljg5MyA2MzMuODEgMzA2LjMzNCA2MzkuODc1 - QzMxMS4xNzQgNjMwLjI4MSAzMTcuMTMxIDYyMC43MjYgMzI0LjMzOSA2MTEuMjIzIiBmaWxsPSIj - ZTBmMjIyIiBmaWxsLXJ1bGU9Im5vbnplcm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9uZSIgdmVj - dG9ybmF0b3I6bGF5ZXJOYW1lPSJwYXRoIi8+CjxwYXRoIGQ9Ik02MzguNDgxIDYyOS41MjRDNjc3 - LjY2OCA1NjEuMTY0IDc1MS4wODggNTI2LjczOSA4MjEuNTI0IDQ4OC44NjZDODIzLjkxIDQ4MS41 - ODkgODI2LjA4NCA0NzQuMjE4IDgyNy45NjMgNDY2LjcyMUM3NTkuNDc4IDUxNC4zNjQgNjc2LjQy - NiA1NDcuODQzIDYyNC42NzkgNjE3LjQzM0M2MDQuOTcxIDY0My45MzYgNTk3Ljc2NyA2NzIuMTkz - IDU5OC41MTEgNzAxLjM0QzYwNi43MTYgNjk5LjA4NCA2MTQuODE0IDY5Ni41NzMgNjIyLjc0NCA2 - OTMuNzA2QzYyMi4yNzYgNjcxLjQ0NiA2MjYuNzkzIDY0OS45MDcgNjM4LjQ4MSA2MjkuNTI0IiBm - aWxsPSIjZTBmMjIyIiBmaWxsLXJ1bGU9Im5vbnplcm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9u - ZSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJwYXRoIi8+CjxwYXRoIGQ9Ik02ODYuNDcxIDY0NC43 - NjRDNjgzLjU3OCA2NTEuNzQ0IDY4MS43IDY1OC44NjIgNjgwLjYxMiA2NjYuMDYyQzczMS43MzYg - NjM1LjA3NiA3NzMuNTgxIDU5MC4zODIgODAxLjIyNyA1MzcuMTI2Qzc0OS41NDkgNTY0LjQzMyA3 - MDYuMTc0IDU5Ny4yMjUgNjg2LjQ3MSA2NDQuNzY0IiBmaWxsPSIjZTBmMjIyIiBmaWxsLXJ1bGU9 - Im5vbnplcm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9uZSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1l - PSJwYXRoIi8+CjxwYXRoIGQ9Ik04MzYuNjY2IDQxMi45MTlDODM3LjMyOCA0MDQuNDQ3IDgzNy43 - NjcgMzk1LjkxNSA4MzcuNzY3IDM4Ny4yOEM4MzcuNzY3IDM3MC42OCA4MzYuNTA3IDM1NC4zODEg - ODM0LjExMyAzMzguNDUxQzgyMi42OCA0NzIuMzY1IDYzNC41MjEgNTIwLjkyMyA1NjMuMzkzIDYy - MC4wNjhDNTQwLjY1OSA2NTEuNzU5IDUzMC41NjIgNjgyLjM3NiA1MjguNjE5IDcxMi42MjNDNTM4 - LjA0MSA3MTIuMTUgNTQ3LjMzNCA3MTEuMTk2IDU1Ni41MzMgNzA5LjkzN0M1NTcuNDc4IDY4Mi44 - MjMgNTY0Ljg1OCA2NTUuNTggNTgxLjg3MSA2MjguMDY3QzYzNy45ODEgNTM3LjMyNSA3NzQuNjg5 - IDQ5OC43MDUgODM2LjY2NiA0MTIuOTE5IiBmaWxsPSIjZTBmMjIyIiBmaWxsLXJ1bGU9Im5vbnpl - cm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9uZSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJwYXRo - Ii8+CjwvZz4KPC9nPgo8ZyBvcGFjaXR5PSIxIiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9Ikdyb3Vw - IDEzIj4KPHBhdGggZD0iTTQzNS4xMTMgNDQ4LjQxM0M0MzUuMTEzIDM3NS43MjMgNDkzLjc5MyAy - NjguNjkyIDUxMS4yMDkgMjY4LjY5MkM1MjguNjI1IDI2OC42OTIgNTg4Ljg3MyAzNzAuOTU3IDU4 - OC44NzMgNDQzLjY0NkM1ODguODczIDUxNi4zMzYgNTMyLjc4NSA2MDAuNzc3IDUxMS4yMDkgNjAw - Ljc3N0M0ODkuNjMzIDYwMC43NzcgNDM1LjExMyA1MjEuMTAzIDQzNS4xMTMgNDQ4LjQxM1oiIGZp - bGw9InVybCgjTGluZWFyR3JhZGllbnQpIiBmaWxsLXJ1bGU9Im5vbnplcm8iIG9wYWNpdHk9IjEi - IHN0cm9rZT0iIzMzMzEyYyIgc3Ryb2tlLWxpbmVjYXA9ImJ1dHQiIHN0cm9rZS1saW5lam9pbj0i - cm91bmQiIHN0cm9rZS13aWR0aD0iMjQuNDgxOSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJPdmFs - IDEiLz4KPHBhdGggZD0iTTQzOS4yMzMgMjY4LjY5MkM0MzkuMjMzIDI2OC42OTIgNDQxLjE1MiAx - OTUuOTMzIDUxMS45OTMgMTk1LjkzM0M1ODIuODM0IDE5NS45MzMgNTg0Ljc1MiAyNjguNjkyIDU4 - NC43NTIgMjY4LjY5Mkw0MzkuMjMzIDI2OC42OTJaIiBmaWxsPSIjZjNjNjIyIiBmaWxsLXJ1bGU9 - Im5vbnplcm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0iIzMzMzEyYyIgc3Ryb2tlLWxpbmVjYXA9ImJ1 - dHQiIHN0cm9rZS1saW5lam9pbj0icm91bmQiIHN0cm9rZS13aWR0aD0iMjQuNDgxOSIgdmVjdG9y - bmF0b3I6bGF5ZXJOYW1lPSJPdmFsIDIiLz4KPHBhdGggZD0iTTQ4MC4xODggMTk1LjkzM0M0ODAu - MTg4IDE5NS45MzMgNDY3LjUxOCAxNjYuMzQ1IDQ0Mi4zMjQgMTc1LjU1OCIgZmlsbD0ibm9uZSIg - b3BhY2l0eT0iMSIgc3Ryb2tlPSIjMzMzMTJjIiBzdHJva2UtbGluZWNhcD0icm91bmQiIHN0cm9r - ZS1saW5lam9pbj0icm91bmQiIHN0cm9rZS13aWR0aD0iMjQuNDgxOSIgdmVjdG9ybmF0b3I6bGF5 - ZXJOYW1lPSJDdXJ2ZSAyIi8+CjxwYXRoIGQ9Ik01NDUuMzM3IDE5NS45MzNDNTQ1LjMzNyAxOTUu - OTMzIDU1OC4wMDYgMTY2LjM0NSA1ODMuMjAxIDE3NS41NTgiIGZpbGw9Im5vbmUiIG9wYWNpdHk9 - IjEiIHN0cm9rZT0iIzMzMzEyYyIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIiBzdHJva2UtbGluZWpv - aW49InJvdW5kIiBzdHJva2Utd2lkdGg9IjI0LjQ4MTkiIHZlY3Rvcm5hdG9yOmxheWVyTmFtZT0i - Q3VydmUgMyIvPgo8ZyBvcGFjaXR5PSIxIiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9Ikdyb3VwIDIi - Pgo8dXNlIGZpbGw9Im5vbmUiIG9wYWNpdHk9IjEiIHN0cm9rZT0iI2YyYWMyMiIgc3Ryb2tlLWxp - bmVjYXA9ImJ1dHQiIHN0cm9rZS1saW5lam9pbj0icm91bmQiIHN0cm9rZS13aWR0aD0iMjQuNDgx - OSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJPdmFsIDUiIHhsaW5rOmhyZWY9IiNGaWxsIi8+Cjxj - bGlwUGF0aCBjbGlwLXJ1bGU9Im5vbnplcm8iIGlkPSJDbGlwUGF0aCI+Cjx1c2UgeGxpbms6aHJl - Zj0iI0ZpbGwiLz4KPC9jbGlwUGF0aD4KPGcgY2xpcC1wYXRoPSJ1cmwoI0NsaXBQYXRoKSI+Cjxw - YXRoIGQ9Ik00MzUuMTEzIDQyNS4yMzhDNDM1LjExMyA0MjUuMjM4IDQ3NC4yMTIgNDM0LjY2MyA1 - MTIgNDM0LjczNUM1NDkuNzg4IDQzNC44MDcgNTg4Ljg3MyA0MjUuNzg1IDU4OC44NzMgNDI1Ljc4 - NSIgZmlsbD0ibm9uZSIgb3BhY2l0eT0iMSIgc3Ryb2tlPSIjZjJkZTIyIiBzdHJva2UtbGluZWNh - cD0iYnV0dCIgc3Ryb2tlLWxpbmVqb2luPSJtaXRlciIgc3Ryb2tlLXdpZHRoPSIyNC40ODE5IiB2 - ZWN0b3JuYXRvcjpsYXllck5hbWU9IkN1cnZlIDQiLz4KPC9nPgo8L2c+CjxnIG9wYWNpdHk9IjEi - IHZlY3Rvcm5hdG9yOmxheWVyTmFtZT0iR3JvdXAgMyI+Cjx1c2UgZmlsbD0ibm9uZSIgb3BhY2l0 - eT0iMSIgc3Ryb2tlPSIjZjNjNjIyIiBzdHJva2UtbGluZWNhcD0iYnV0dCIgc3Ryb2tlLWxpbmVq - b2luPSJyb3VuZCIgc3Ryb2tlLXdpZHRoPSIyNC40ODE5IiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9 - Ik92YWwgNCIgeGxpbms6aHJlZj0iI0ZpbGxfMiIvPgo8Y2xpcFBhdGggY2xpcC1ydWxlPSJub256 - ZXJvIiBpZD0iQ2xpcFBhdGhfMiI+Cjx1c2UgeGxpbms6aHJlZj0iI0ZpbGxfMiIvPgo8L2NsaXBQ - YXRoPgo8ZyBjbGlwLXBhdGg9InVybCgjQ2xpcFBhdGhfMikiPgo8cGF0aCBkPSJNNDM5LjIzMyA0 - ODEuNjUzQzQzOS4yMzMgNDgxLjY1MyA0NzUuNjIgNDkzLjg5MiA1MTIgNDkzLjg5MkM1NDguMzgg - NDkzLjg5MiA1ODQuNzUyIDQ4MS42NTMgNTg0Ljc1MiA0ODEuNjUzIiBmaWxsPSJub25lIiBvcGFj - aXR5PSIxIiBzdHJva2U9IiNmM2M2MjIiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIgc3Ryb2tlLWxp - bmVqb2luPSJtaXRlciIgc3Ryb2tlLXdpZHRoPSIyNC40ODE5IiB2ZWN0b3JuYXRvcjpsYXllck5h - bWU9IkN1cnZlIDUiLz4KPC9nPgo8L2c+CjxnIG9wYWNpdHk9IjEiIHZlY3Rvcm5hdG9yOmxheWVy - TmFtZT0iR3JvdXAgNCI+Cjx1c2UgZmlsbD0ibm9uZSIgb3BhY2l0eT0iMSIgc3Ryb2tlPSIjZjJk - ZTIyIiBzdHJva2UtbGluZWNhcD0iYnV0dCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIgc3Ryb2tl - LXdpZHRoPSIyNC40ODE5IiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9Ik92YWwgMyIgeGxpbms6aHJl - Zj0iI0ZpbGxfMyIvPgo8Y2xpcFBhdGggY2xpcC1ydWxlPSJub256ZXJvIiBpZD0iQ2xpcFBhdGhf - MyI+Cjx1c2UgeGxpbms6aHJlZj0iI0ZpbGxfMyIvPgo8L2NsaXBQYXRoPgo8ZyBjbGlwLXBhdGg9 - InVybCgjQ2xpcFBhdGhfMykiPgo8cGF0aCBkPSJNNDYxLjI1NiA1NDAuMTc0QzQ2MS4yNTYgNTQw - LjE3NCA0ODcuNDU5IDU0OS41ODYgNTExLjk5MyA1NDkuNTIzQzUzNi41MjcgNTQ5LjQ2MSA1NTku - MzkxIDUzOS45MjUgNTU5LjM5MSA1MzkuOTI1IiBmaWxsPSJub25lIiBvcGFjaXR5PSIxIiBzdHJv - a2U9IiNmMmRlMjIiIHN0cm9rZS1saW5lY2FwPSJzcXVhcmUiIHN0cm9rZS1saW5lam9pbj0ibWl0 - ZXIiIHN0cm9rZS13aWR0aD0iMjQuNDgxOSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJDdXJ2ZSA2 - Ii8+CjwvZz4KPC9nPgo8ZyBvcGFjaXR5PSIxIiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9Ikdyb3Vw - IDEiPgo8dXNlIGZpbGw9Im5vbmUiIG9wYWNpdHk9IjEiIHN0cm9rZT0iI2YzYzYyMiIgc3Ryb2tl - LWxpbmVjYXA9ImJ1dHQiIHN0cm9rZS1saW5lam9pbj0ibWl0ZXIiIHN0cm9rZS13aWR0aD0iMjQu - NDgxOSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJDdXJ2ZSA3IiB4bGluazpocmVmPSIjRmlsbF80 - Ii8+CjxjbGlwUGF0aCBjbGlwLXJ1bGU9Im5vbnplcm8iIGlkPSJDbGlwUGF0aF80Ij4KPHVzZSB4 - bGluazpocmVmPSIjRmlsbF80Ii8+CjwvY2xpcFBhdGg+CjxnIGNsaXAtcGF0aD0idXJsKCNDbGlw - UGF0aF80KSI+CjxwYXRoIGQ9Ik00MzUuMTEzIDQ0OC40MTNDNDM1LjExMyAzNzUuNzIzIDQ5My43 - OTMgMjY4LjY5MiA1MTEuMjA5IDI2OC42OTJDNTI4LjYyNSAyNjguNjkyIDU4OC44NzMgMzcwLjk1 - NyA1ODguODczIDQ0My42NDZDNTg4Ljg3MyA1MTYuMzM2IDUzMi43ODUgNjAwLjc3NyA1MTEuMjA5 - IDYwMC43NzdDNDg5LjYzMyA2MDAuNzc3IDQzNS4xMTMgNTIxLjEwMyA0MzUuMTEzIDQ0OC40MTNa - IiBmaWxsPSJub25lIiBvcGFjaXR5PSIxIiBzdHJva2U9IiNmM2M2MjIiIHN0cm9rZS1saW5lY2Fw - PSJidXR0IiBzdHJva2UtbGluZWpvaW49InJvdW5kIiBzdHJva2Utd2lkdGg9IjI0LjQ4MTkiIHZl - Y3Rvcm5hdG9yOmxheWVyTmFtZT0iT3ZhbCA0Ii8+CjwvZz4KPC9nPgo8cGF0aCBkPSJNNDM1LjEx - MyA0NDguNDEzQzQzNS4xMTMgMzc1LjcyMyA0OTMuNzkzIDI2OC42OTIgNTExLjIwOSAyNjguNjky - QzUyOC42MjUgMjY4LjY5MiA1ODguODczIDM3MC45NTcgNTg4Ljg3MyA0NDMuNjQ2QzU4OC44NzMg - NTE2LjMzNiA1MzIuNzg1IDYwMC43NzcgNTExLjIwOSA2MDAuNzc3QzQ4OS42MzMgNjAwLjc3NyA0 - MzUuMTEzIDUyMS4xMDMgNDM1LjExMyA0NDguNDEzWiIgZmlsbD0ibm9uZSIgb3BhY2l0eT0iMSIg - c3Ryb2tlPSIjMzMzMTJjIiBzdHJva2UtbGluZWNhcD0iYnV0dCIgc3Ryb2tlLWxpbmVqb2luPSJy - b3VuZCIgc3Ryb2tlLXdpZHRoPSIyNC40ODE5IiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9Ik92YWwg - MyIvPgo8cGF0aCBkPSJNNDkwLjQ1MyAyNjkuNDQ2QzQ1MC4xMjUgMjY5LjgwMyAzNjEuNTgxIDI4 - NS40MjggMjk4LjI2OSAzMjUuNTU2QzE5NS4wNDcgMzkwLjk3OCAyNjQuMjI0IDQ2OS4wODkgMzI2 - LjIxMSA0NjguMjI5QzQyMi4xODYgNDY2Ljg5NyA1MTEuOTg5IDMwMC4yNjIgNTExLjk4OSAyNzMu - ODU2QzUxMS45ODkgMjcyLjExMiA1MDkuMTIgMjcwLjgzOSA1MDMuOTYxIDI3MC4xMkM1MDAuNDU1 - IDI2OS42MzEgNDk1Ljg5MiAyNjkuMzk4IDQ5MC40NTMgMjY5LjQ0NlpNNTExLjk4OSAyNzMuODU2 - QzUxMS45ODkgMzAwLjI2MiA2MDEuNzkyIDQ2Ni44OTcgNjk3Ljc2NyA0NjguMjI5Qzc1OS43NTQg - NDY5LjA4OSA4MjguOTYzIDM5MC45NzggNzI1Ljc0MSAzMjUuNTU2QzY1NC4yMzkgMjgwLjIzNyA1 - NTAuNTA5IDI2Ni4xOTIgNTIwLjQ0NSAyNzAuMDc2QzUxNS4wMTYgMjcwLjc3NyA1MTEuOTg5IDI3 - Mi4wNjQgNTExLjk4OSAyNzMuODU2WiIgZmlsbD0iI2ZmZmZmZiIgZmlsbC1ydWxlPSJub256ZXJv - IiBvcGFjaXR5PSIxIiBzdHJva2U9IiMzMzMxMmMiIHN0cm9rZS1saW5lY2FwPSJidXR0IiBzdHJv - a2UtbGluZWpvaW49InJvdW5kIiBzdHJva2Utd2lkdGg9IjI0LjQ4MTkiIHZlY3Rvcm5hdG9yOmxh - eWVyTmFtZT0iQ3VydmUgMSIvPgo8L2c+CjxnIG9wYWNpdHk9IjEiIHZlY3Rvcm5hdG9yOmxheWVy - TmFtZT0iR3JvdXAgMTQiPgo8cGF0aCBkPSJNMTU0LjkxOSA5MjQuNTMzQzE0NS4zOTQgOTI0LjUz - MyAxMzcuMTYxIDkyMi41MTcgMTMwLjIyMSA5MTguNDg1QzEyMy4yOCA5MTQuNDU0IDExNy45Njkg - OTA4LjI2NCAxMTQuMjg2IDg5OS45MThDMTEwLjYwMyA4OTEuNTcxIDEwOC43NjEgODgwLjk5MiAx - MDguNzYxIDg2OC4xODJDMTA4Ljc2MSA4NTUuMzQzIDExMC42ODQgODQ0Ljc4NCAxMTQuNTMxIDgz - Ni41MDZDMTE4LjM3NyA4MjguMjI4IDEyMy43ODQgODIyLjA2IDEzMC43NTEgODE4LjAwMkMxMzcu - NzE4IDgxMy45NDQgMTQ1Ljc3NCA4MTEuOTE0IDE1NC45MTkgODExLjkxNEMxNjUuMjY1IDgxMS45 - MTQgMTc0LjU1MSA4MTQuMjIyIDE4Mi43NzggODE4LjgzN0MxOTEuMDA1IDgyMy40NTIgMTk3LjUy - IDgyOS45NjcgMjAyLjMyNCA4MzguMzgxQzIwNy4xMjggODQ2Ljc5NiAyMDkuNTMgODU2LjczIDIw - OS41MyA4NjguMTgyQzIwOS41MyA4NzkuNjM1IDIwNy4xMjggODg5LjU2OSAyMDIuMzI0IDg5Ny45 - ODNDMTk3LjUyIDkwNi4zOTggMTkxLjAwNSA5MTIuOTI3IDE4Mi43NzggOTE3LjU2OUMxNzQuNTUx - IDkyMi4yMTIgMTY1LjI2NSA5MjQuNTMzIDE1NC45MTkgOTI0LjUzM1pNOTAuMzA5NyA5MjIuOTg3 - TDkwLjMwOTcgNzcxLjkxM0wxMjIuMDcyIDc3MS45MTNMMTIyLjA3MiA4MzUuNTcxTDEyMC4wMzYg - ODY4LjA1OEwxMjAuNTcgOTAwLjU0OUwxMjAuNTcgOTIyLjk4N0w5MC4zMDk3IDkyMi45ODdaTTE0 - OS40NTQgODk4LjUyNkMxNTQuNzMgODk4LjUyNiAxNTkuNDYzIDg5Ny4zMjIgMTYzLjY1MiA4OTQu - OTE1QzE2Ny44NDEgODkyLjUwOCAxNzEuMTc5IDg4OS4wMDcgMTczLjY2OCA4ODQuNDEyQzE3Ni4x - NTcgODc5LjgxNyAxNzcuNDAxIDg3NC40MDcgMTc3LjQwMSA4NjguMTgyQzE3Ny40MDEgODYxLjgy - MiAxNzYuMTU3IDg1Ni4zOTEgMTczLjY2OCA4NTEuODkxQzE3MS4xNzkgODQ3LjM5IDE2Ny44NDEg - ODQzLjkzNyAxNjMuNjUyIDg0MS41MjlDMTU5LjQ2MyA4MzkuMTIyIDE1NC43MyA4MzcuOTE5IDE0 - OS40NTQgODM3LjkxOUMxNDQuMTc3IDgzNy45MTkgMTM5LjQ0NCA4MzkuMTIyIDEzNS4yNTQgODQx - LjUyOUMxMzEuMDY0IDg0My45MzcgMTI3LjcyNSA4NDcuMzkgMTI1LjIzNiA4NTEuODkxQzEyMi43 - NDcgODU2LjM5MSAxMjEuNTAzIDg2MS44MjIgMTIxLjUwMyA4NjguMTgyQzEyMS41MDMgODc0LjQw - NyAxMjIuNzQ3IDg3OS44MTcgMTI1LjIzNiA4ODQuNDEyQzEyNy43MjUgODg5LjAwNyAxMzEuMDY0 - IDg5Mi41MDggMTM1LjI1NCA4OTQuOTE1QzEzOS40NDQgODk3LjMyMiAxNDQuMTc3IDg5OC41MjYg - MTQ5LjQ1NCA4OTguNTI2WiIgZmlsbD0iI2YzYzYyMiIgZmlsbC1ydWxlPSJub256ZXJvIiBvcGFj - aXR5PSIxIiBzdHJva2U9Im5vbmUiLz4KPHBhdGggZD0iTTI5NS40NzYgOTI0LjUzM0MyODYuMzMx - IDkyNC41MzMgMjc4LjI3NSA5MjIuNTA0IDI3MS4zMDggOTE4LjQ0NkMyNjQuMzQxIDkxNC4zODcg - MjU4LjkzNCA5MDguMTk4IDI1NS4wODggODk5Ljg3OEMyNTEuMjQyIDg5MS41NTggMjQ5LjMxOSA4 - ODEuMDE5IDI0OS4zMTkgODY4LjI2MkMyNDkuMzE5IDg1NS4zMTYgMjUxLjE2IDg0NC43MDQgMjU0 - Ljg0MyA4MzYuNDI1QzI1OC41MjYgODI4LjE0NiAyNjMuODM4IDgyMS45OTEgMjcwLjc3OCA4MTcu - OTYxQzI3Ny43MTkgODEzLjkzIDI4NS45NTEgODExLjkxNCAyOTUuNDc2IDgxMS45MTRDMzA1Ljgy - MiA4MTEuOTE0IDMxNS4xMDggODE0LjIzNSAzMjMuMzM1IDgxOC44NzdDMzMxLjU2MiA4MjMuNTE4 - IDMzOC4wNzcgODMwLjA0NiAzNDIuODgxIDgzOC40NjFDMzQ3LjY4NSA4NDYuODc2IDM1MC4wODcg - ODU2LjgwOSAzNTAuMDg3IDg2OC4yNjJDMzUwLjA4NyA4NzkuNzE3IDM0Ny42ODUgODg5LjY1MSAz - NDIuODgxIDg5OC4wNjZDMzM4LjA3NyA5MDYuNDgxIDMzMS41NjIgOTEyLjk5NSAzMjMuMzM1IDkx - Ny42MUMzMTUuMTA4IDkyMi4yMjUgMzA1LjgyMiA5MjQuNTMzIDI5NS40NzYgOTI0LjUzM1pNMjMw - Ljg2NyA5NjIuNDg2TDIzMC44NjcgODEzLjQ1N0wyNjEuMTI4IDgxMy40NTdMMjYxLjEyOCA4MzUu - ODk1TDI2MC41OTMgODY4LjM4NkwyNjIuNjI5IDkwMC44NzdMMjYyLjYyOSA5NjIuNDg2TDIzMC44 - NjcgOTYyLjQ4NlpNMjkwLjAxMSA4OTguNTI2QzI5NS4yODggODk4LjUyNiAzMDAuMDIgODk3LjMy - MiAzMDQuMjA5IDg5NC45MTVDMzA4LjM5OCA4OTIuNTA4IDMxMS43MzYgODg5LjAyIDMxNC4yMjUg - ODg0LjQ1MkMzMTYuNzE0IDg3OS44ODMgMzE3Ljk1OSA4NzQuNDg3IDMxNy45NTkgODY4LjI2MkMz - MTcuOTU5IDg2MS45MDEgMzE2LjcxNCA4NTYuNDU4IDMxNC4yMjUgODUxLjkzMUMzMTEuNzM2IDg0 - Ny40MDQgMzA4LjM5OCA4NDMuOTM3IDMwNC4yMDkgODQxLjUyOUMzMDAuMDIgODM5LjEyMiAyOTUu - Mjg4IDgzNy45MTkgMjkwLjAxMSA4MzcuOTE5QzI4NC43MzQgODM3LjkxOSAyODAuMDAxIDgzOS4x - MjIgMjc1LjgxMSA4NDEuNTI5QzI3MS42MjEgODQzLjkzNyAyNjguMjgyIDg0Ny40MDQgMjY1Ljc5 - MyA4NTEuOTMxQzI2My4zMDQgODU2LjQ1OCAyNjIuMDYgODYxLjkwMSAyNjIuMDYgODY4LjI2MkMy - NjIuMDYgODc0LjQ4NyAyNjMuMzA0IDg3OS44ODMgMjY1Ljc5MyA4ODQuNDUyQzI2OC4yODIgODg5 - LjAyIDI3MS42MjEgODkyLjUwOCAyNzUuODExIDg5NC45MTVDMjgwLjAwMSA4OTcuMzIyIDI4NC43 - MzQgODk4LjUyNiAyOTAuMDExIDg5OC41MjZaIiBmaWxsPSIjZjNjNjIyIiBmaWxsLXJ1bGU9Im5v - bnplcm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9uZSIvPgo8cGF0aCBkPSJNMzc1LjI4NiA5MjIu - OTg3TDM3NS4yODYgODEwLjk3QzM3NS4yODYgNzk4LjYzIDM3OC45NDEgNzg4Ljc3OCAzODYuMjQ5 - IDc4MS40MTRDMzkzLjU1OCA3NzQuMDQ5IDQwNC4wMDYgNzcwLjM2NiA0MTcuNTk1IDc3MC4zNjZD - NDIyLjE1MiA3NzAuMzY2IDQyNi41ODEgNzcwLjgyOCA0MzAuODggNzcxLjc1MkM0MzUuMTc5IDc3 - Mi42NzYgNDM4LjgyIDc3NC4xMjkgNDQxLjgwNCA3NzYuMTEyTDQzMy41MTQgNzk5LjExQzQzMS43 - NzUgNzk3LjkzOSA0MjkuNzk4IDc5NyA0MjcuNTgyIDc5Ni4yOTNDNDI1LjM2NyA3OTUuNTg1IDQy - My4wNjMgNzk1LjIzMiA0MjAuNjcxIDc5NS4yMzJDNDE2LjAyMiA3OTUuMjMyIDQxMi40MzMgNzk2 - LjU1NyA0MDkuOTA1IDc5OS4yMDhDNDA3LjM3NyA4MDEuODU5IDQwNi4xMTMgODA1Ljg3NCA0MDYu - MTEzIDgxMS4yNTNMNDA2LjExMyA4MjEuNDYyTDQwNy4wNDkgODM1LjExNkw0MDcuMDQ5IDkyMi45 - ODdMMzc1LjI4NiA5MjIuOTg3Wk0zNTguMjk1IDg0MC4yNkwzNTguMjk1IDgxNS44ODJMNDM0LjI3 - NCA4MTUuODgyTDQzNC4yNzQgODQwLjI2TDM1OC4yOTUgODQwLjI2WiIgZmlsbD0iI2YzYzYyMiIg - ZmlsbC1ydWxlPSJub256ZXJvIiBvcGFjaXR5PSIxIiBzdHJva2U9Im5vbmUiLz4KPC9nPgo8ZyBv - cGFjaXR5PSIxIiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9Ikdyb3VwIDE1Ij4KPHBhdGggZD0iTTYy - Mi45MDYgODExLjkxNEM2MzEuNTY4IDgxMS45MTQgNjM5LjI1OCA4MTMuNjI0IDY0NS45NzcgODE3 - LjA0NEM2NTIuNjk2IDgyMC40NjQgNjU3Ljk2OSA4MjUuNzM3IDY2MS43OTYgODMyLjg2MkM2NjUu - NjIzIDgzOS45ODcgNjY3LjUzNyA4NDkuMTQgNjY3LjUzNyA4NjAuMzIxTDY2Ny41MzcgOTIyLjk4 - N0w2MzUuNzc1IDkyMi45ODdMNjM1Ljc3NSA4NjUuMDY4QzYzNS43NzUgODU2LjI2MiA2MzMuOTQ1 - IDg0OS43NjUgNjMwLjI4NyA4NDUuNTc4QzYyNi42MjggODQxLjM5IDYyMS41MzMgODM5LjI5NiA2 - MTUuMDAxIDgzOS4yOTZDNjEwLjMyIDgzOS4yOTYgNjA2LjE1NyA4NDAuMzQzIDYwMi41MTEgODQy - LjQzOEM1OTguODY1IDg0NC41MzIgNTk2LjAyOSA4NDcuNjczIDU5NC4wMDMgODUxLjg2MUM1OTEu - OTc2IDg1Ni4wNDggNTkwLjk2MyA4NjEuNDMyIDU5MC45NjMgODY4LjAxMUw1OTAuOTYzIDkyMi45 - ODdMNTU5LjE5NyA5MjIuOTg3TDU1OS4xOTcgODY1LjA2OEM1NTkuMTk3IDg1Ni4yNjIgNTU3LjM5 - NSA4NDkuNzY1IDU1My43OTEgODQ1LjU3OEM1NTAuMTg2IDg0MS4zOSA1NDUuMDY0IDgzOS4yOTYg - NTM4LjQyMyA4MzkuMjk2QzUzMy43NDUgODM5LjI5NiA1MjkuNTgyIDg0MC4zNDMgNTI1LjkzNSA4 - NDIuNDM4QzUyMi4yODggODQ0LjUzMiA1MTkuNDUyIDg0Ny42NzMgNTE3LjQyNSA4NTEuODYxQzUx - NS4zOTkgODU2LjA0OCA1MTQuMzg1IDg2MS40MzIgNTE0LjM4NSA4NjguMDExTDUxNC4zODUgOTIy - Ljk4N0w0ODIuNjIzIDkyMi45ODdMNDgyLjYyMyA4MTMuNDU3TDUxMi44ODQgODEzLjQ1N0w1MTIu - ODg0IDg0My4zMzZMNTA3LjExOSA4MzQuNjIzQzUxMC45MTggODI3LjE4OSA1MTYuMzI1IDgyMS41 - NDYgNTIzLjM0MiA4MTcuNjkzQzUzMC4zNTkgODEzLjg0MSA1MzguMzM0IDgxMS45MTQgNTQ3LjI2 - NyA4MTEuOTE0QzU1Ny4yODYgODExLjkxNCA1NjYuMDYzIDgxNC40MzkgNTczLjU5NiA4MTkuNDg4 - QzU4MS4xMjkgODI0LjUzNiA1ODYuMTQ0IDgzMi4zMTIgNTg4LjY0IDg0Mi44MTVMNTc3LjQyIDgz - OS43MTZDNTgxLjA4MSA4MzEuMjI0IDU4Ni45MzYgODI0LjQ2NyA1OTQuOTg3IDgxOS40NDZDNjAz - LjAzOCA4MTQuNDI1IDYxMi4zNDQgODExLjkxNCA2MjIuOTA2IDgxMS45MTRaIiBmaWxsPSIjOTk5 - OTk5IiBmaWxsLXJ1bGU9Im5vbnplcm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9uZSIvPgo8cGF0 - aCBkPSJNNzYzLjIzOCA5MjIuOTg3TDc2My4yMzggOTAxLjY0TDc2MS4zMzMgODk2LjgzNkw3NjEu - MzMzIDg1OC42MzhDNzYxLjMzMyA4NTEuODE2IDc1OS4yNiA4NDYuNTI5IDc1NS4xMTUgODQyLjc3 - OEM3NTAuOTY5IDgzOS4wMjcgNzQ0LjU5MSA4MzcuMTUyIDczNS45ODEgODM3LjE1MkM3MzAuMjIg - ODM3LjE1MiA3MjQuNTE2IDgzOC4wNTUgNzE4Ljg2NyA4MzkuODYxQzcxMy4yMTggODQxLjY2NyA3 - MDguMzk3IDg0NC4xMzIgNzA0LjQwMyA4NDcuMjU2TDY5My4wNjIgODI1LjE1MkM2OTkuMDg1IDgy - MC44MzQgNzA2LjI5NiA4MTcuNTQ4IDcxNC42OTQgODE1LjI5NUM3MjMuMDkzIDgxMy4wNDEgNzMx - LjYzNiA4MTEuOTE0IDc0MC4zMjMgODExLjkxNEM3NTcuMDg3IDgxMS45MTQgNzcwLjA3MyA4MTUu - ODQyIDc3OS4yODIgODIzLjY5NkM3ODguNDkgODMxLjU1MSA3OTMuMDk1IDg0My43OTMgNzkzLjA5 - NSA4NjAuNDIzTDc5My4wOTUgOTIyLjk4N0w3NjMuMjM4IDkyMi45ODdaTTcyOS45NTggOTI0LjUz - M0M3MjEuNDM5IDkyNC41MzMgNzE0LjEyNSA5MjMuMDk1IDcwOC4wMTcgOTIwLjIxOEM3MDEuOTA5 - IDkxNy4zNDEgNjk3LjIzMiA5MTMuMzkyIDY5My45ODcgOTA4LjM3QzY5MC43NDIgOTAzLjM0OSA2 - ODkuMTIgODk3LjY5IDY4OS4xMiA4OTEuMzkzQzY4OS4xMiA4ODQuOTM3IDY5MC43MDYgODc5LjI1 - OSA2OTMuODc5IDg3NC4zNTlDNjk3LjA1MiA4NjkuNDU5IDcwMi4wOTggODY1LjYxNyA3MDkuMDE3 - IDg2Mi44MzNDNzE1LjkzNyA4NjAuMDQ5IDcyNC45NjEgODU4LjY1OCA3MzYuMDg5IDg1OC42NThM - NzY1LjA3NCA4NTguNjU4TDc2NS4wNzQgODc3LjE2Nkw3MzkuNjExIDg3Ny4xNjZDNzMyLjEwMyA4 - NzcuMTY2IDcyNi45NjMgODc4LjM3NyA3MjQuMTkxIDg4MC43OThDNzIxLjQxOSA4ODMuMjE5IDcy - MC4wMzMgODg2LjMxOCA3MjAuMDMzIDg5MC4wOTVDNzIwLjAzMyA4OTQuMDY2IDcyMS42MTEgODk3 - LjI0NyA3MjQuNzY3IDg5OS42NDFDNzI3LjkyMiA5MDIuMDM0IDczMi4yNjIgOTAzLjIzMSA3Mzcu - Nzg0IDkwMy4yMzFDNzQzLjExOCA5MDMuMjMxIDc0Ny45MDcgOTAxLjk5MyA3NTIuMTUxIDg5OS41 - MThDNzU2LjM5NSA4OTcuMDQzIDc1OS40NTYgODkzLjMxOCA3NjEuMzMzIDg4OC4zNDJMNzY2LjEz - IDkwMy4wOTFDNzYzLjg3NSA5MTAuMDYgNzU5LjY5MyA5MTUuMzc2IDc1My41ODEgOTE5LjAzOUM3 - NDcuNDcgOTIyLjcwMiA3MzkuNTk2IDkyNC41MzMgNzI5Ljk1OCA5MjQuNTMzWiIgZmlsbD0iIzk5 - OTk5OSIgZmlsbC1ydWxlPSJub256ZXJvIiBvcGFjaXR5PSIxIiBzdHJva2U9Im5vbmUiLz4KPHBh - dGggZD0iTTg4OC4yMDcgODExLjkxNEM4OTYuOTIyIDgxMS45MTQgOTA0LjY5OSA4MTMuNjI0IDkx - MS41NCA4MTcuMDQ0QzkxOC4zODEgODIwLjQ2NCA5MjMuNzgzIDgyNS43MzcgOTI3Ljc0NiA4MzIu - ODYyQzkzMS43MDkgODM5Ljk4NyA5MzMuNjkgODQ5LjE0IDkzMy42OSA4NjAuMzIxTDkzMy42OSA5 - MjIuOTg3TDkwMS45MjggOTIyLjk4N0w5MDEuOTI4IDg2NS4wNjhDOTAxLjkyOCA4NTYuMjYyIDg5 - OS45OSA4NDkuNzY1IDg5Ni4xMTMgODQ1LjU3OEM4OTIuMjM2IDg0MS4zOSA4ODYuODAxIDgzOS4y - OTYgODc5LjgwOCA4MzkuMjk2Qzg3NC43MiA4MzkuMjk2IDg3MC4xNzcgODQwLjM1NiA4NjYuMTgg - ODQyLjQ3N0M4NjIuMTgyIDg0NC41OTggODU5LjA3NSA4NDcuODIgODU2Ljg1OCA4NTIuMTQyQzg1 - NC42NDIgODU2LjQ2NSA4NTMuNTM0IDg2Mi4wMjMgODUzLjUzNCA4NjguODE5TDg1My41MzQgOTIy - Ljk4N0w4MjEuNzcyIDkyMi45ODdMODIxLjc3MiA4MTMuNDU3TDg1Mi4wMzIgODEzLjQ1N0w4NTIu - MDMyIDg0My44MjNMODQ2LjM0NyA4MzQuNzAyQzg1MC4yODEgODI3LjI3MSA4NTUuOTA3IDgyMS42 - MTUgODYzLjIyMyA4MTcuNzM1Qzg3MC41MzkgODEzLjg1NSA4NzguODY3IDgxMS45MTQgODg4LjIw - NyA4MTEuOTE0WiIgZmlsbD0iIzk5OTk5OSIgZmlsbC1ydWxlPSJub256ZXJvIiBvcGFjaXR5PSIx - IiBzdHJva2U9Im5vbmUiLz4KPC9nPgo8L2c+Cjwvc3ZnPgo= - mediatype: image/svg+xml + - base64data: | + PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+Cjwh + RE9DVFlQRSBzdmcgUFVCTElDICItLy9XM0MvL0RURCBTVkcgMS4xLy9FTiIgImh0dHA6Ly93d3cu + dzMub3JnL0dyYXBoaWNzL1NWRy8xLjEvRFREL3N2ZzExLmR0ZCI+CjwhLS0gQ3JlYXRlZCB3aXRo + IFZlY3Rvcm5hdG9yIChodHRwOi8vdmVjdG9ybmF0b3IuaW8vKSAtLT4KPHN2ZyBoZWlnaHQ9IjEw + MCUiIHN0cm9rZS1taXRlcmxpbWl0PSIxMCIgc3R5bGU9ImZpbGwtcnVsZTpub256ZXJvO2NsaXAt + cnVsZTpldmVub2RkO3N0cm9rZS1saW5lY2FwOnJvdW5kO3N0cm9rZS1saW5lam9pbjpyb3VuZDsi + IHZlcnNpb249IjEuMSIgdmlld0JveD0iMCAwIDEwMjQgMTAyNCIgd2lkdGg9IjEwMCUiIHhtbDpz + cGFjZT0icHJlc2VydmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgeG1sbnM6 + dmVjdG9ybmF0b3I9Imh0dHA6Ly92ZWN0b3JuYXRvci5pbyIgeG1sbnM6eGxpbms9Imh0dHA6Ly93 + d3cudzMub3JnLzE5OTkveGxpbmsiPgo8ZGVmcz4KPGxpbmVhckdyYWRpZW50IGdyYWRpZW50VHJh + bnNmb3JtPSJtYXRyaXgoMS40NTY4MyAxLjQ1NjgzIC0xLjQ1NjgzIDEuNDU2ODMgNzkxLjI0NCAt + NzA0LjEzMykiIGdyYWRpZW50VW5pdHM9InVzZXJTcGFjZU9uVXNlIiBpZD0iTGluZWFyR3JhZGll + bnQiIHgxPSIzNjguNDYyIiB4Mj0iMjQyLjMzMSIgeTE9IjQyNy4wNTMiIHkyPSI1NTMuNjE0Ij4K + PHN0b3Agb2Zmc2V0PSIwLjQyMjU5MiIgc3RvcC1jb2xvcj0iIzMzMzEyYyIvPgo8c3RvcCBvZmZz + ZXQ9IjEiIHN0b3AtY29sb3I9IiNmM2M2MjIiLz4KPC9saW5lYXJHcmFkaWVudD4KPHBhdGggZD0i + TTQzNS4xMTMgNDQ4LjQxM0M0MzUuMTEzIDM3NS43MjMgNDkzLjc5MyAyNjguNjkyIDUxMS4yMDkg + MjY4LjY5MkM1MjguNjI1IDI2OC42OTIgNTg4Ljg3MyAzNzAuOTU3IDU4OC44NzMgNDQzLjY0NkM1 + ODguODczIDUxNi4zMzYgNTMyLjc4NSA2MDAuNzc3IDUxMS4yMDkgNjAwLjc3N0M0ODkuNjMzIDYw + MC43NzcgNDM1LjExMyA1MjEuMTAzIDQzNS4xMTMgNDQ4LjQxM1oiIGlkPSJGaWxsIi8+CjxwYXRo + IGQ9Ik00MzUuMTEzIDQ0OC40MTNDNDM1LjExMyAzNzUuNzIzIDQ5My43OTMgMjY4LjY5MiA1MTEu + MjA5IDI2OC42OTJDNTI4LjYyNSAyNjguNjkyIDU4OC44NzMgMzcwLjk1NyA1ODguODczIDQ0My42 + NDZDNTg4Ljg3MyA1MTYuMzM2IDUzMi43ODUgNjAwLjc3NyA1MTEuMjA5IDYwMC43NzdDNDg5LjYz + MyA2MDAuNzc3IDQzNS4xMTMgNTIxLjEwMyA0MzUuMTEzIDQ0OC40MTNaIiBpZD0iRmlsbF8yIi8+ + CjxwYXRoIGQ9Ik00MzUuMTEzIDQ0OC40MTNDNDM1LjExMyAzNzUuNzIzIDQ5My43OTMgMjY4LjY5 + MiA1MTEuMjA5IDI2OC42OTJDNTI4LjYyNSAyNjguNjkyIDU4OC44NzMgMzcwLjk1NyA1ODguODcz + IDQ0My42NDZDNTg4Ljg3MyA1MTYuMzM2IDUzMi43ODUgNjAwLjc3NyA1MTEuMjA5IDYwMC43NzdD + NDg5LjYzMyA2MDAuNzc3IDQzNS4xMTMgNTIxLjEwMyA0MzUuMTEzIDQ0OC40MTNaIiBpZD0iRmls + bF8zIi8+CjxwYXRoIGQ9Ik00MzUuMTIgMzY4LjgzOEM0MzUuMTIgMzY4LjgzOCA0NzQuMjE5IDM3 + OC4yNjMgNTEyLjAwNyAzNzguMzM1QzU0OS43OTYgMzc4LjQwNyA1ODguODggMzY5LjM4NSA1ODgu + ODggMzY5LjM4NSIgaWQ9IkZpbGxfNCIvPgo8L2RlZnM+CjxnIGlkPSJMYXllci0xIiB2ZWN0b3Ju + YXRvcjpsYXllck5hbWU9IkxheWVyIDEiPgo8ZyBvcGFjaXR5PSIxIiB2ZWN0b3JuYXRvcjpsYXll + ck5hbWU9Ikdyb3VwIDEyIj4KPHBhdGggZD0iTTE4Ni4yMzMgMzg3LjI3OEMxODYuMjMzIDIwNy4z + NjQgMzMyLjA4NCA2MS41MTM5IDUxMS45OTggNjEuNTEzOUM2OTEuOTE2IDYxLjUxMzkgODM3Ljc2 + NyAyMDcuMzY0IDgzNy43NjcgMzg3LjI3OEM4MzcuNzY3IDU2Ny4xOTMgNjkxLjkxNiA3MTMuMDQz + IDUxMS45OTggNzEzLjA0M0MzMzIuMDg0IDcxMy4wNDMgMTg2LjIzMyA1NjcuMTkzIDE4Ni4yMzMg + Mzg3LjI3OCIgZmlsbD0iI2YyOGYyMiIgZmlsbC1ydWxlPSJub256ZXJvIiBvcGFjaXR5PSIxIiBz + dHJva2U9Im5vbmUiIHZlY3Rvcm5hdG9yOmxheWVyTmFtZT0icGF0aCIvPgo8ZyBvcGFjaXR5PSIx + IiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9Ikdyb3VwIDEzIj4KPHBhdGggZD0iTTIwMS4zMzEgNDg1 + LjQ4NUMyMDQuODQ3IDQ4MS41MzQgMjA4LjE4MiA0NzcuNTkzIDIxMS4yOTQgNDczLjY2QzI3MS4z + ODUgMzk3LjYzNiAyNDkuMTUgMzQzLjc2OSAxOTguMTU5IDI5OS45OTFDMTkwLjQ0MyAzMjcuNzg0 + IDE4Ni4yMzUgMzU3LjAzIDE4Ni4yMzUgMzg3LjI3N0MxODYuMjM1IDQyMS41MDkgMTkxLjU0NCA0 + NTQuNDkgMjAxLjMzMSA0ODUuNDg1IiBmaWxsPSIjZTBmMjIyIiBmaWxsLXJ1bGU9Im5vbnplcm8i + IG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9uZSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJwYXRoIi8+ + CjxwYXRoIGQ9Ik00MjEuMTYzIDQwNS4wOThDNDQzLjY2IDMyMy4yNzkgMzQxLjE0MSAyNTIuNjQx + IDI0Ni4xNzkgMTk5LjA4QzIzOSAyMDkuMTk4IDIzMi4zNDIgMjE5LjcwMyAyMjYuMzM4IDIzMC42 + MjlDMzA5LjI2NSAyNzguMTY5IDM5MC43NjkgMzQyLjM1OSAzNTYuNjU0IDQyMy4zMDFDMzM0LjIy + NSA0NzYuNTA1IDI3OC43MTMgNTEzLjMwNyAyMzIuNzA4IDU1NS4wMDFDMjM4LjYxNCA1NjQuODE4 + IDI0NS4wMjcgNTc0LjI5MiAyNTEuOTA0IDU4My4zOTZDMzA5Ljk0NCA1MjQuODAxIDM5OS4zNTMg + NDg0LjQyNyA0MjEuMTYzIDQwNS4wOTgiIGZpbGw9IiNlMGYyMjIiIGZpbGwtcnVsZT0ibm9uemVy + byIgb3BhY2l0eT0iMSIgc3Ryb2tlPSJub25lIiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9InBhdGgi + Lz4KPHBhdGggZD0iTTU5Ny45MjMgMzIzLjgwN0M2MjkuMjkyIDQ0Mi45OTkgNDU0LjQwMiA1MTku + MzQxIDM4OC45NiA1OTIuMzA1QzM2Ni4zMDQgNjE3LjU2NiAzNDguODAxIDYzOS4xODcgMzM5LjMw + NiA2NjMuNDY0QzM0OC43NTggNjY5LjM4NyAzNTguNTE1IDY3NC44NTggMzY4LjU4NiA2NzkuODAx + QzM3My43NzQgNjU2LjU2NSAzODQuNTUgNjMyLjg1MSA0MDIuODQgNjA4LjIzNUM0NzEuODAyIDUx + NS40MTcgNjUwLjg2NSA0NTUuNTg1IDYyNi4zMTMgMzE4LjkwM0M2MDcuNzE4IDIxNS4zODYgNDk4 + LjE3NiAxNjEuODQyIDQ2MS45MjQgNjUuMzQ5N0M0MzguOTk3IDY4Ljg4NzIgNDE2Ljg5NSA3NC44 + NzQ4IDM5NS44MDggODIuOTI5OEM0NDcuMzAxIDE3My44MTMgNTcwLjgyNiAyMjAuODQxIDU5Ny45 + MjMgMzIzLjgwNyIgZmlsbD0iI2UwZjIyMiIgZmlsbC1ydWxlPSJub256ZXJvIiBvcGFjaXR5PSIx + IiBzdHJva2U9Im5vbmUiIHZlY3Rvcm5hdG9yOmxheWVyTmFtZT0icGF0aCIvPgo8cGF0aCBkPSJN + NzA1LjQ4MSAzMDQuMzA1Qzc0MC4zNzkgNDYwLjIwOSA1MTkuNjMxIDUyMy4yNjUgNDQ2LjAyNiA2 + MzAuMzQ1QzQzMC40ODQgNjUyLjk0OSA0MjMuMDc4IDY3Ni4zNzEgNDIxLjI1OSA3MDAuMTQxQzQz + NC4zMjYgNzAzLjkyMyA0NDcuNjk4IDcwNi45NzUgNDYxLjM4NCA3MDkuMTExQzQ2Mi43NzIgNjg0 + LjQ2OSA0NjkuOCA2NjAuMjI2IDQ4NS4xMTUgNjM2Ljg4N0M1NTguODAxIDUyNC41ODUgNzg2Ljk3 + MiA0NjguMDg4IDc2MC4xMDggMzA4LjUxM0M3NDcuMzg1IDIzMi45MjcgNzAwLjQ0MyAxNzQuMzU0 + IDY3NS4zNDEgMTA1LjQ2NUM2NTAuMDI4IDkwLjc2MDggNjIyLjU5NiA3OS4yOTMgNTkzLjU0OSA3 + MS44MDUzQzYxOC4xODYgMTU1Ljg1OSA2ODUuNzY0IDIxNi4yMzcgNzA1LjQ4MSAzMDQuMzA1IiBm + aWxsPSIjZTBmMjIyIiBmaWxsLXJ1bGU9Im5vbnplcm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9u + ZSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJwYXRoIi8+CjxwYXRoIGQ9Ik0zMjQuMzM5IDYxMS4y + MjNDMzgxLjUzNyA1MzUuNzk2IDU2My4wOCA0NjMuODc3IDU1Mi43MyAzNTIuNzQ4QzU0My41OTIg + MjU0LjY0OCA0MDIuNjYzIDE5NC4wNzYgMzE2LjE1NSAxMjYuOTYyQzMwNi40NDEgMTM0LjI4MiAy + OTcuMiAxNDIuMTc0IDI4OC4zNzUgMTUwLjUxM0MzODUuNjg1IDIxMS44NTQgNTE1Ljk3MSAyNzcu + NDc2IDUxMy40MzUgMzY1LjIzOUM1MTAuMDk5IDQ4MC42MzIgMzQ3LjM3NCA1MzMuNDMyIDI4NC44 + ODEgNjIwLjcxM0MyOTEuNzU0IDYyNy40MDIgMjk4Ljg5MyA2MzMuODEgMzA2LjMzNCA2MzkuODc1 + QzMxMS4xNzQgNjMwLjI4MSAzMTcuMTMxIDYyMC43MjYgMzI0LjMzOSA2MTEuMjIzIiBmaWxsPSIj + ZTBmMjIyIiBmaWxsLXJ1bGU9Im5vbnplcm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9uZSIgdmVj + dG9ybmF0b3I6bGF5ZXJOYW1lPSJwYXRoIi8+CjxwYXRoIGQ9Ik02MzguNDgxIDYyOS41MjRDNjc3 + LjY2OCA1NjEuMTY0IDc1MS4wODggNTI2LjczOSA4MjEuNTI0IDQ4OC44NjZDODIzLjkxIDQ4MS41 + ODkgODI2LjA4NCA0NzQuMjE4IDgyNy45NjMgNDY2LjcyMUM3NTkuNDc4IDUxNC4zNjQgNjc2LjQy + NiA1NDcuODQzIDYyNC42NzkgNjE3LjQzM0M2MDQuOTcxIDY0My45MzYgNTk3Ljc2NyA2NzIuMTkz + IDU5OC41MTEgNzAxLjM0QzYwNi43MTYgNjk5LjA4NCA2MTQuODE0IDY5Ni41NzMgNjIyLjc0NCA2 + OTMuNzA2QzYyMi4yNzYgNjcxLjQ0NiA2MjYuNzkzIDY0OS45MDcgNjM4LjQ4MSA2MjkuNTI0IiBm + aWxsPSIjZTBmMjIyIiBmaWxsLXJ1bGU9Im5vbnplcm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9u + ZSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJwYXRoIi8+CjxwYXRoIGQ9Ik02ODYuNDcxIDY0NC43 + NjRDNjgzLjU3OCA2NTEuNzQ0IDY4MS43IDY1OC44NjIgNjgwLjYxMiA2NjYuMDYyQzczMS43MzYg + NjM1LjA3NiA3NzMuNTgxIDU5MC4zODIgODAxLjIyNyA1MzcuMTI2Qzc0OS41NDkgNTY0LjQzMyA3 + MDYuMTc0IDU5Ny4yMjUgNjg2LjQ3MSA2NDQuNzY0IiBmaWxsPSIjZTBmMjIyIiBmaWxsLXJ1bGU9 + Im5vbnplcm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9uZSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1l + PSJwYXRoIi8+CjxwYXRoIGQ9Ik04MzYuNjY2IDQxMi45MTlDODM3LjMyOCA0MDQuNDQ3IDgzNy43 + NjcgMzk1LjkxNSA4MzcuNzY3IDM4Ny4yOEM4MzcuNzY3IDM3MC42OCA4MzYuNTA3IDM1NC4zODEg + ODM0LjExMyAzMzguNDUxQzgyMi42OCA0NzIuMzY1IDYzNC41MjEgNTIwLjkyMyA1NjMuMzkzIDYy + MC4wNjhDNTQwLjY1OSA2NTEuNzU5IDUzMC41NjIgNjgyLjM3NiA1MjguNjE5IDcxMi42MjNDNTM4 + LjA0MSA3MTIuMTUgNTQ3LjMzNCA3MTEuMTk2IDU1Ni41MzMgNzA5LjkzN0M1NTcuNDc4IDY4Mi44 + MjMgNTY0Ljg1OCA2NTUuNTggNTgxLjg3MSA2MjguMDY3QzYzNy45ODEgNTM3LjMyNSA3NzQuNjg5 + IDQ5OC43MDUgODM2LjY2NiA0MTIuOTE5IiBmaWxsPSIjZTBmMjIyIiBmaWxsLXJ1bGU9Im5vbnpl + cm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9uZSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJwYXRo + Ii8+CjwvZz4KPC9nPgo8ZyBvcGFjaXR5PSIxIiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9Ikdyb3Vw + IDEzIj4KPHBhdGggZD0iTTQzNS4xMTMgNDQ4LjQxM0M0MzUuMTEzIDM3NS43MjMgNDkzLjc5MyAy + NjguNjkyIDUxMS4yMDkgMjY4LjY5MkM1MjguNjI1IDI2OC42OTIgNTg4Ljg3MyAzNzAuOTU3IDU4 + OC44NzMgNDQzLjY0NkM1ODguODczIDUxNi4zMzYgNTMyLjc4NSA2MDAuNzc3IDUxMS4yMDkgNjAw + Ljc3N0M0ODkuNjMzIDYwMC43NzcgNDM1LjExMyA1MjEuMTAzIDQzNS4xMTMgNDQ4LjQxM1oiIGZp + bGw9InVybCgjTGluZWFyR3JhZGllbnQpIiBmaWxsLXJ1bGU9Im5vbnplcm8iIG9wYWNpdHk9IjEi + IHN0cm9rZT0iIzMzMzEyYyIgc3Ryb2tlLWxpbmVjYXA9ImJ1dHQiIHN0cm9rZS1saW5lam9pbj0i + cm91bmQiIHN0cm9rZS13aWR0aD0iMjQuNDgxOSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJPdmFs + IDEiLz4KPHBhdGggZD0iTTQzOS4yMzMgMjY4LjY5MkM0MzkuMjMzIDI2OC42OTIgNDQxLjE1MiAx + OTUuOTMzIDUxMS45OTMgMTk1LjkzM0M1ODIuODM0IDE5NS45MzMgNTg0Ljc1MiAyNjguNjkyIDU4 + NC43NTIgMjY4LjY5Mkw0MzkuMjMzIDI2OC42OTJaIiBmaWxsPSIjZjNjNjIyIiBmaWxsLXJ1bGU9 + Im5vbnplcm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0iIzMzMzEyYyIgc3Ryb2tlLWxpbmVjYXA9ImJ1 + dHQiIHN0cm9rZS1saW5lam9pbj0icm91bmQiIHN0cm9rZS13aWR0aD0iMjQuNDgxOSIgdmVjdG9y + bmF0b3I6bGF5ZXJOYW1lPSJPdmFsIDIiLz4KPHBhdGggZD0iTTQ4MC4xODggMTk1LjkzM0M0ODAu + MTg4IDE5NS45MzMgNDY3LjUxOCAxNjYuMzQ1IDQ0Mi4zMjQgMTc1LjU1OCIgZmlsbD0ibm9uZSIg + b3BhY2l0eT0iMSIgc3Ryb2tlPSIjMzMzMTJjIiBzdHJva2UtbGluZWNhcD0icm91bmQiIHN0cm9r + ZS1saW5lam9pbj0icm91bmQiIHN0cm9rZS13aWR0aD0iMjQuNDgxOSIgdmVjdG9ybmF0b3I6bGF5 + ZXJOYW1lPSJDdXJ2ZSAyIi8+CjxwYXRoIGQ9Ik01NDUuMzM3IDE5NS45MzNDNTQ1LjMzNyAxOTUu + OTMzIDU1OC4wMDYgMTY2LjM0NSA1ODMuMjAxIDE3NS41NTgiIGZpbGw9Im5vbmUiIG9wYWNpdHk9 + IjEiIHN0cm9rZT0iIzMzMzEyYyIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIiBzdHJva2UtbGluZWpv + aW49InJvdW5kIiBzdHJva2Utd2lkdGg9IjI0LjQ4MTkiIHZlY3Rvcm5hdG9yOmxheWVyTmFtZT0i + Q3VydmUgMyIvPgo8ZyBvcGFjaXR5PSIxIiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9Ikdyb3VwIDIi + Pgo8dXNlIGZpbGw9Im5vbmUiIG9wYWNpdHk9IjEiIHN0cm9rZT0iI2YyYWMyMiIgc3Ryb2tlLWxp + bmVjYXA9ImJ1dHQiIHN0cm9rZS1saW5lam9pbj0icm91bmQiIHN0cm9rZS13aWR0aD0iMjQuNDgx + OSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJPdmFsIDUiIHhsaW5rOmhyZWY9IiNGaWxsIi8+Cjxj + bGlwUGF0aCBjbGlwLXJ1bGU9Im5vbnplcm8iIGlkPSJDbGlwUGF0aCI+Cjx1c2UgeGxpbms6aHJl + Zj0iI0ZpbGwiLz4KPC9jbGlwUGF0aD4KPGcgY2xpcC1wYXRoPSJ1cmwoI0NsaXBQYXRoKSI+Cjxw + YXRoIGQ9Ik00MzUuMTEzIDQyNS4yMzhDNDM1LjExMyA0MjUuMjM4IDQ3NC4yMTIgNDM0LjY2MyA1 + MTIgNDM0LjczNUM1NDkuNzg4IDQzNC44MDcgNTg4Ljg3MyA0MjUuNzg1IDU4OC44NzMgNDI1Ljc4 + NSIgZmlsbD0ibm9uZSIgb3BhY2l0eT0iMSIgc3Ryb2tlPSIjZjJkZTIyIiBzdHJva2UtbGluZWNh + cD0iYnV0dCIgc3Ryb2tlLWxpbmVqb2luPSJtaXRlciIgc3Ryb2tlLXdpZHRoPSIyNC40ODE5IiB2 + ZWN0b3JuYXRvcjpsYXllck5hbWU9IkN1cnZlIDQiLz4KPC9nPgo8L2c+CjxnIG9wYWNpdHk9IjEi + IHZlY3Rvcm5hdG9yOmxheWVyTmFtZT0iR3JvdXAgMyI+Cjx1c2UgZmlsbD0ibm9uZSIgb3BhY2l0 + eT0iMSIgc3Ryb2tlPSIjZjNjNjIyIiBzdHJva2UtbGluZWNhcD0iYnV0dCIgc3Ryb2tlLWxpbmVq + b2luPSJyb3VuZCIgc3Ryb2tlLXdpZHRoPSIyNC40ODE5IiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9 + Ik92YWwgNCIgeGxpbms6aHJlZj0iI0ZpbGxfMiIvPgo8Y2xpcFBhdGggY2xpcC1ydWxlPSJub256 + ZXJvIiBpZD0iQ2xpcFBhdGhfMiI+Cjx1c2UgeGxpbms6aHJlZj0iI0ZpbGxfMiIvPgo8L2NsaXBQ + YXRoPgo8ZyBjbGlwLXBhdGg9InVybCgjQ2xpcFBhdGhfMikiPgo8cGF0aCBkPSJNNDM5LjIzMyA0 + ODEuNjUzQzQzOS4yMzMgNDgxLjY1MyA0NzUuNjIgNDkzLjg5MiA1MTIgNDkzLjg5MkM1NDguMzgg + NDkzLjg5MiA1ODQuNzUyIDQ4MS42NTMgNTg0Ljc1MiA0ODEuNjUzIiBmaWxsPSJub25lIiBvcGFj + aXR5PSIxIiBzdHJva2U9IiNmM2M2MjIiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIgc3Ryb2tlLWxp + bmVqb2luPSJtaXRlciIgc3Ryb2tlLXdpZHRoPSIyNC40ODE5IiB2ZWN0b3JuYXRvcjpsYXllck5h + bWU9IkN1cnZlIDUiLz4KPC9nPgo8L2c+CjxnIG9wYWNpdHk9IjEiIHZlY3Rvcm5hdG9yOmxheWVy + TmFtZT0iR3JvdXAgNCI+Cjx1c2UgZmlsbD0ibm9uZSIgb3BhY2l0eT0iMSIgc3Ryb2tlPSIjZjJk + ZTIyIiBzdHJva2UtbGluZWNhcD0iYnV0dCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIgc3Ryb2tl + LXdpZHRoPSIyNC40ODE5IiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9Ik92YWwgMyIgeGxpbms6aHJl + Zj0iI0ZpbGxfMyIvPgo8Y2xpcFBhdGggY2xpcC1ydWxlPSJub256ZXJvIiBpZD0iQ2xpcFBhdGhf + MyI+Cjx1c2UgeGxpbms6aHJlZj0iI0ZpbGxfMyIvPgo8L2NsaXBQYXRoPgo8ZyBjbGlwLXBhdGg9 + InVybCgjQ2xpcFBhdGhfMykiPgo8cGF0aCBkPSJNNDYxLjI1NiA1NDAuMTc0QzQ2MS4yNTYgNTQw + LjE3NCA0ODcuNDU5IDU0OS41ODYgNTExLjk5MyA1NDkuNTIzQzUzNi41MjcgNTQ5LjQ2MSA1NTku + MzkxIDUzOS45MjUgNTU5LjM5MSA1MzkuOTI1IiBmaWxsPSJub25lIiBvcGFjaXR5PSIxIiBzdHJv + a2U9IiNmMmRlMjIiIHN0cm9rZS1saW5lY2FwPSJzcXVhcmUiIHN0cm9rZS1saW5lam9pbj0ibWl0 + ZXIiIHN0cm9rZS13aWR0aD0iMjQuNDgxOSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJDdXJ2ZSA2 + Ii8+CjwvZz4KPC9nPgo8ZyBvcGFjaXR5PSIxIiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9Ikdyb3Vw + IDEiPgo8dXNlIGZpbGw9Im5vbmUiIG9wYWNpdHk9IjEiIHN0cm9rZT0iI2YzYzYyMiIgc3Ryb2tl + LWxpbmVjYXA9ImJ1dHQiIHN0cm9rZS1saW5lam9pbj0ibWl0ZXIiIHN0cm9rZS13aWR0aD0iMjQu + NDgxOSIgdmVjdG9ybmF0b3I6bGF5ZXJOYW1lPSJDdXJ2ZSA3IiB4bGluazpocmVmPSIjRmlsbF80 + Ii8+CjxjbGlwUGF0aCBjbGlwLXJ1bGU9Im5vbnplcm8iIGlkPSJDbGlwUGF0aF80Ij4KPHVzZSB4 + bGluazpocmVmPSIjRmlsbF80Ii8+CjwvY2xpcFBhdGg+CjxnIGNsaXAtcGF0aD0idXJsKCNDbGlw + UGF0aF80KSI+CjxwYXRoIGQ9Ik00MzUuMTEzIDQ0OC40MTNDNDM1LjExMyAzNzUuNzIzIDQ5My43 + OTMgMjY4LjY5MiA1MTEuMjA5IDI2OC42OTJDNTI4LjYyNSAyNjguNjkyIDU4OC44NzMgMzcwLjk1 + NyA1ODguODczIDQ0My42NDZDNTg4Ljg3MyA1MTYuMzM2IDUzMi43ODUgNjAwLjc3NyA1MTEuMjA5 + IDYwMC43NzdDNDg5LjYzMyA2MDAuNzc3IDQzNS4xMTMgNTIxLjEwMyA0MzUuMTEzIDQ0OC40MTNa + IiBmaWxsPSJub25lIiBvcGFjaXR5PSIxIiBzdHJva2U9IiNmM2M2MjIiIHN0cm9rZS1saW5lY2Fw + PSJidXR0IiBzdHJva2UtbGluZWpvaW49InJvdW5kIiBzdHJva2Utd2lkdGg9IjI0LjQ4MTkiIHZl + Y3Rvcm5hdG9yOmxheWVyTmFtZT0iT3ZhbCA0Ii8+CjwvZz4KPC9nPgo8cGF0aCBkPSJNNDM1LjEx + MyA0NDguNDEzQzQzNS4xMTMgMzc1LjcyMyA0OTMuNzkzIDI2OC42OTIgNTExLjIwOSAyNjguNjky + QzUyOC42MjUgMjY4LjY5MiA1ODguODczIDM3MC45NTcgNTg4Ljg3MyA0NDMuNjQ2QzU4OC44NzMg + NTE2LjMzNiA1MzIuNzg1IDYwMC43NzcgNTExLjIwOSA2MDAuNzc3QzQ4OS42MzMgNjAwLjc3NyA0 + MzUuMTEzIDUyMS4xMDMgNDM1LjExMyA0NDguNDEzWiIgZmlsbD0ibm9uZSIgb3BhY2l0eT0iMSIg + c3Ryb2tlPSIjMzMzMTJjIiBzdHJva2UtbGluZWNhcD0iYnV0dCIgc3Ryb2tlLWxpbmVqb2luPSJy + b3VuZCIgc3Ryb2tlLXdpZHRoPSIyNC40ODE5IiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9Ik92YWwg + MyIvPgo8cGF0aCBkPSJNNDkwLjQ1MyAyNjkuNDQ2QzQ1MC4xMjUgMjY5LjgwMyAzNjEuNTgxIDI4 + NS40MjggMjk4LjI2OSAzMjUuNTU2QzE5NS4wNDcgMzkwLjk3OCAyNjQuMjI0IDQ2OS4wODkgMzI2 + LjIxMSA0NjguMjI5QzQyMi4xODYgNDY2Ljg5NyA1MTEuOTg5IDMwMC4yNjIgNTExLjk4OSAyNzMu + ODU2QzUxMS45ODkgMjcyLjExMiA1MDkuMTIgMjcwLjgzOSA1MDMuOTYxIDI3MC4xMkM1MDAuNDU1 + IDI2OS42MzEgNDk1Ljg5MiAyNjkuMzk4IDQ5MC40NTMgMjY5LjQ0NlpNNTExLjk4OSAyNzMuODU2 + QzUxMS45ODkgMzAwLjI2MiA2MDEuNzkyIDQ2Ni44OTcgNjk3Ljc2NyA0NjguMjI5Qzc1OS43NTQg + NDY5LjA4OSA4MjguOTYzIDM5MC45NzggNzI1Ljc0MSAzMjUuNTU2QzY1NC4yMzkgMjgwLjIzNyA1 + NTAuNTA5IDI2Ni4xOTIgNTIwLjQ0NSAyNzAuMDc2QzUxNS4wMTYgMjcwLjc3NyA1MTEuOTg5IDI3 + Mi4wNjQgNTExLjk4OSAyNzMuODU2WiIgZmlsbD0iI2ZmZmZmZiIgZmlsbC1ydWxlPSJub256ZXJv + IiBvcGFjaXR5PSIxIiBzdHJva2U9IiMzMzMxMmMiIHN0cm9rZS1saW5lY2FwPSJidXR0IiBzdHJv + a2UtbGluZWpvaW49InJvdW5kIiBzdHJva2Utd2lkdGg9IjI0LjQ4MTkiIHZlY3Rvcm5hdG9yOmxh + eWVyTmFtZT0iQ3VydmUgMSIvPgo8L2c+CjxnIG9wYWNpdHk9IjEiIHZlY3Rvcm5hdG9yOmxheWVy + TmFtZT0iR3JvdXAgMTQiPgo8cGF0aCBkPSJNMTU0LjkxOSA5MjQuNTMzQzE0NS4zOTQgOTI0LjUz + MyAxMzcuMTYxIDkyMi41MTcgMTMwLjIyMSA5MTguNDg1QzEyMy4yOCA5MTQuNDU0IDExNy45Njkg + OTA4LjI2NCAxMTQuMjg2IDg5OS45MThDMTEwLjYwMyA4OTEuNTcxIDEwOC43NjEgODgwLjk5MiAx + MDguNzYxIDg2OC4xODJDMTA4Ljc2MSA4NTUuMzQzIDExMC42ODQgODQ0Ljc4NCAxMTQuNTMxIDgz + Ni41MDZDMTE4LjM3NyA4MjguMjI4IDEyMy43ODQgODIyLjA2IDEzMC43NTEgODE4LjAwMkMxMzcu + NzE4IDgxMy45NDQgMTQ1Ljc3NCA4MTEuOTE0IDE1NC45MTkgODExLjkxNEMxNjUuMjY1IDgxMS45 + MTQgMTc0LjU1MSA4MTQuMjIyIDE4Mi43NzggODE4LjgzN0MxOTEuMDA1IDgyMy40NTIgMTk3LjUy + IDgyOS45NjcgMjAyLjMyNCA4MzguMzgxQzIwNy4xMjggODQ2Ljc5NiAyMDkuNTMgODU2LjczIDIw + OS41MyA4NjguMTgyQzIwOS41MyA4NzkuNjM1IDIwNy4xMjggODg5LjU2OSAyMDIuMzI0IDg5Ny45 + ODNDMTk3LjUyIDkwNi4zOTggMTkxLjAwNSA5MTIuOTI3IDE4Mi43NzggOTE3LjU2OUMxNzQuNTUx + IDkyMi4yMTIgMTY1LjI2NSA5MjQuNTMzIDE1NC45MTkgOTI0LjUzM1pNOTAuMzA5NyA5MjIuOTg3 + TDkwLjMwOTcgNzcxLjkxM0wxMjIuMDcyIDc3MS45MTNMMTIyLjA3MiA4MzUuNTcxTDEyMC4wMzYg + ODY4LjA1OEwxMjAuNTcgOTAwLjU0OUwxMjAuNTcgOTIyLjk4N0w5MC4zMDk3IDkyMi45ODdaTTE0 + OS40NTQgODk4LjUyNkMxNTQuNzMgODk4LjUyNiAxNTkuNDYzIDg5Ny4zMjIgMTYzLjY1MiA4OTQu + OTE1QzE2Ny44NDEgODkyLjUwOCAxNzEuMTc5IDg4OS4wMDcgMTczLjY2OCA4ODQuNDEyQzE3Ni4x + NTcgODc5LjgxNyAxNzcuNDAxIDg3NC40MDcgMTc3LjQwMSA4NjguMTgyQzE3Ny40MDEgODYxLjgy + MiAxNzYuMTU3IDg1Ni4zOTEgMTczLjY2OCA4NTEuODkxQzE3MS4xNzkgODQ3LjM5IDE2Ny44NDEg + ODQzLjkzNyAxNjMuNjUyIDg0MS41MjlDMTU5LjQ2MyA4MzkuMTIyIDE1NC43MyA4MzcuOTE5IDE0 + OS40NTQgODM3LjkxOUMxNDQuMTc3IDgzNy45MTkgMTM5LjQ0NCA4MzkuMTIyIDEzNS4yNTQgODQx + LjUyOUMxMzEuMDY0IDg0My45MzcgMTI3LjcyNSA4NDcuMzkgMTI1LjIzNiA4NTEuODkxQzEyMi43 + NDcgODU2LjM5MSAxMjEuNTAzIDg2MS44MjIgMTIxLjUwMyA4NjguMTgyQzEyMS41MDMgODc0LjQw + NyAxMjIuNzQ3IDg3OS44MTcgMTI1LjIzNiA4ODQuNDEyQzEyNy43MjUgODg5LjAwNyAxMzEuMDY0 + IDg5Mi41MDggMTM1LjI1NCA4OTQuOTE1QzEzOS40NDQgODk3LjMyMiAxNDQuMTc3IDg5OC41MjYg + MTQ5LjQ1NCA4OTguNTI2WiIgZmlsbD0iI2YzYzYyMiIgZmlsbC1ydWxlPSJub256ZXJvIiBvcGFj + aXR5PSIxIiBzdHJva2U9Im5vbmUiLz4KPHBhdGggZD0iTTI5NS40NzYgOTI0LjUzM0MyODYuMzMx + IDkyNC41MzMgMjc4LjI3NSA5MjIuNTA0IDI3MS4zMDggOTE4LjQ0NkMyNjQuMzQxIDkxNC4zODcg + MjU4LjkzNCA5MDguMTk4IDI1NS4wODggODk5Ljg3OEMyNTEuMjQyIDg5MS41NTggMjQ5LjMxOSA4 + ODEuMDE5IDI0OS4zMTkgODY4LjI2MkMyNDkuMzE5IDg1NS4zMTYgMjUxLjE2IDg0NC43MDQgMjU0 + Ljg0MyA4MzYuNDI1QzI1OC41MjYgODI4LjE0NiAyNjMuODM4IDgyMS45OTEgMjcwLjc3OCA4MTcu + OTYxQzI3Ny43MTkgODEzLjkzIDI4NS45NTEgODExLjkxNCAyOTUuNDc2IDgxMS45MTRDMzA1Ljgy + MiA4MTEuOTE0IDMxNS4xMDggODE0LjIzNSAzMjMuMzM1IDgxOC44NzdDMzMxLjU2MiA4MjMuNTE4 + IDMzOC4wNzcgODMwLjA0NiAzNDIuODgxIDgzOC40NjFDMzQ3LjY4NSA4NDYuODc2IDM1MC4wODcg + ODU2LjgwOSAzNTAuMDg3IDg2OC4yNjJDMzUwLjA4NyA4NzkuNzE3IDM0Ny42ODUgODg5LjY1MSAz + NDIuODgxIDg5OC4wNjZDMzM4LjA3NyA5MDYuNDgxIDMzMS41NjIgOTEyLjk5NSAzMjMuMzM1IDkx + Ny42MUMzMTUuMTA4IDkyMi4yMjUgMzA1LjgyMiA5MjQuNTMzIDI5NS40NzYgOTI0LjUzM1pNMjMw + Ljg2NyA5NjIuNDg2TDIzMC44NjcgODEzLjQ1N0wyNjEuMTI4IDgxMy40NTdMMjYxLjEyOCA4MzUu + ODk1TDI2MC41OTMgODY4LjM4NkwyNjIuNjI5IDkwMC44NzdMMjYyLjYyOSA5NjIuNDg2TDIzMC44 + NjcgOTYyLjQ4NlpNMjkwLjAxMSA4OTguNTI2QzI5NS4yODggODk4LjUyNiAzMDAuMDIgODk3LjMy + MiAzMDQuMjA5IDg5NC45MTVDMzA4LjM5OCA4OTIuNTA4IDMxMS43MzYgODg5LjAyIDMxNC4yMjUg + ODg0LjQ1MkMzMTYuNzE0IDg3OS44ODMgMzE3Ljk1OSA4NzQuNDg3IDMxNy45NTkgODY4LjI2MkMz + MTcuOTU5IDg2MS45MDEgMzE2LjcxNCA4NTYuNDU4IDMxNC4yMjUgODUxLjkzMUMzMTEuNzM2IDg0 + Ny40MDQgMzA4LjM5OCA4NDMuOTM3IDMwNC4yMDkgODQxLjUyOUMzMDAuMDIgODM5LjEyMiAyOTUu + Mjg4IDgzNy45MTkgMjkwLjAxMSA4MzcuOTE5QzI4NC43MzQgODM3LjkxOSAyODAuMDAxIDgzOS4x + MjIgMjc1LjgxMSA4NDEuNTI5QzI3MS42MjEgODQzLjkzNyAyNjguMjgyIDg0Ny40MDQgMjY1Ljc5 + MyA4NTEuOTMxQzI2My4zMDQgODU2LjQ1OCAyNjIuMDYgODYxLjkwMSAyNjIuMDYgODY4LjI2MkMy + NjIuMDYgODc0LjQ4NyAyNjMuMzA0IDg3OS44ODMgMjY1Ljc5MyA4ODQuNDUyQzI2OC4yODIgODg5 + LjAyIDI3MS42MjEgODkyLjUwOCAyNzUuODExIDg5NC45MTVDMjgwLjAwMSA4OTcuMzIyIDI4NC43 + MzQgODk4LjUyNiAyOTAuMDExIDg5OC41MjZaIiBmaWxsPSIjZjNjNjIyIiBmaWxsLXJ1bGU9Im5v + bnplcm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9uZSIvPgo8cGF0aCBkPSJNMzc1LjI4NiA5MjIu + OTg3TDM3NS4yODYgODEwLjk3QzM3NS4yODYgNzk4LjYzIDM3OC45NDEgNzg4Ljc3OCAzODYuMjQ5 + IDc4MS40MTRDMzkzLjU1OCA3NzQuMDQ5IDQwNC4wMDYgNzcwLjM2NiA0MTcuNTk1IDc3MC4zNjZD + NDIyLjE1MiA3NzAuMzY2IDQyNi41ODEgNzcwLjgyOCA0MzAuODggNzcxLjc1MkM0MzUuMTc5IDc3 + Mi42NzYgNDM4LjgyIDc3NC4xMjkgNDQxLjgwNCA3NzYuMTEyTDQzMy41MTQgNzk5LjExQzQzMS43 + NzUgNzk3LjkzOSA0MjkuNzk4IDc5NyA0MjcuNTgyIDc5Ni4yOTNDNDI1LjM2NyA3OTUuNTg1IDQy + My4wNjMgNzk1LjIzMiA0MjAuNjcxIDc5NS4yMzJDNDE2LjAyMiA3OTUuMjMyIDQxMi40MzMgNzk2 + LjU1NyA0MDkuOTA1IDc5OS4yMDhDNDA3LjM3NyA4MDEuODU5IDQwNi4xMTMgODA1Ljg3NCA0MDYu + MTEzIDgxMS4yNTNMNDA2LjExMyA4MjEuNDYyTDQwNy4wNDkgODM1LjExNkw0MDcuMDQ5IDkyMi45 + ODdMMzc1LjI4NiA5MjIuOTg3Wk0zNTguMjk1IDg0MC4yNkwzNTguMjk1IDgxNS44ODJMNDM0LjI3 + NCA4MTUuODgyTDQzNC4yNzQgODQwLjI2TDM1OC4yOTUgODQwLjI2WiIgZmlsbD0iI2YzYzYyMiIg + ZmlsbC1ydWxlPSJub256ZXJvIiBvcGFjaXR5PSIxIiBzdHJva2U9Im5vbmUiLz4KPC9nPgo8ZyBv + cGFjaXR5PSIxIiB2ZWN0b3JuYXRvcjpsYXllck5hbWU9Ikdyb3VwIDE1Ij4KPHBhdGggZD0iTTYy + Mi45MDYgODExLjkxNEM2MzEuNTY4IDgxMS45MTQgNjM5LjI1OCA4MTMuNjI0IDY0NS45NzcgODE3 + LjA0NEM2NTIuNjk2IDgyMC40NjQgNjU3Ljk2OSA4MjUuNzM3IDY2MS43OTYgODMyLjg2MkM2NjUu + NjIzIDgzOS45ODcgNjY3LjUzNyA4NDkuMTQgNjY3LjUzNyA4NjAuMzIxTDY2Ny41MzcgOTIyLjk4 + N0w2MzUuNzc1IDkyMi45ODdMNjM1Ljc3NSA4NjUuMDY4QzYzNS43NzUgODU2LjI2MiA2MzMuOTQ1 + IDg0OS43NjUgNjMwLjI4NyA4NDUuNTc4QzYyNi42MjggODQxLjM5IDYyMS41MzMgODM5LjI5NiA2 + MTUuMDAxIDgzOS4yOTZDNjEwLjMyIDgzOS4yOTYgNjA2LjE1NyA4NDAuMzQzIDYwMi41MTEgODQy + LjQzOEM1OTguODY1IDg0NC41MzIgNTk2LjAyOSA4NDcuNjczIDU5NC4wMDMgODUxLjg2MUM1OTEu + OTc2IDg1Ni4wNDggNTkwLjk2MyA4NjEuNDMyIDU5MC45NjMgODY4LjAxMUw1OTAuOTYzIDkyMi45 + ODdMNTU5LjE5NyA5MjIuOTg3TDU1OS4xOTcgODY1LjA2OEM1NTkuMTk3IDg1Ni4yNjIgNTU3LjM5 + NSA4NDkuNzY1IDU1My43OTEgODQ1LjU3OEM1NTAuMTg2IDg0MS4zOSA1NDUuMDY0IDgzOS4yOTYg + NTM4LjQyMyA4MzkuMjk2QzUzMy43NDUgODM5LjI5NiA1MjkuNTgyIDg0MC4zNDMgNTI1LjkzNSA4 + NDIuNDM4QzUyMi4yODggODQ0LjUzMiA1MTkuNDUyIDg0Ny42NzMgNTE3LjQyNSA4NTEuODYxQzUx + NS4zOTkgODU2LjA0OCA1MTQuMzg1IDg2MS40MzIgNTE0LjM4NSA4NjguMDExTDUxNC4zODUgOTIy + Ljk4N0w0ODIuNjIzIDkyMi45ODdMNDgyLjYyMyA4MTMuNDU3TDUxMi44ODQgODEzLjQ1N0w1MTIu + ODg0IDg0My4zMzZMNTA3LjExOSA4MzQuNjIzQzUxMC45MTggODI3LjE4OSA1MTYuMzI1IDgyMS41 + NDYgNTIzLjM0MiA4MTcuNjkzQzUzMC4zNTkgODEzLjg0MSA1MzguMzM0IDgxMS45MTQgNTQ3LjI2 + NyA4MTEuOTE0QzU1Ny4yODYgODExLjkxNCA1NjYuMDYzIDgxNC40MzkgNTczLjU5NiA4MTkuNDg4 + QzU4MS4xMjkgODI0LjUzNiA1ODYuMTQ0IDgzMi4zMTIgNTg4LjY0IDg0Mi44MTVMNTc3LjQyIDgz + OS43MTZDNTgxLjA4MSA4MzEuMjI0IDU4Ni45MzYgODI0LjQ2NyA1OTQuOTg3IDgxOS40NDZDNjAz + LjAzOCA4MTQuNDI1IDYxMi4zNDQgODExLjkxNCA2MjIuOTA2IDgxMS45MTRaIiBmaWxsPSIjOTk5 + OTk5IiBmaWxsLXJ1bGU9Im5vbnplcm8iIG9wYWNpdHk9IjEiIHN0cm9rZT0ibm9uZSIvPgo8cGF0 + aCBkPSJNNzYzLjIzOCA5MjIuOTg3TDc2My4yMzggOTAxLjY0TDc2MS4zMzMgODk2LjgzNkw3NjEu + MzMzIDg1OC42MzhDNzYxLjMzMyA4NTEuODE2IDc1OS4yNiA4NDYuNTI5IDc1NS4xMTUgODQyLjc3 + OEM3NTAuOTY5IDgzOS4wMjcgNzQ0LjU5MSA4MzcuMTUyIDczNS45ODEgODM3LjE1MkM3MzAuMjIg + ODM3LjE1MiA3MjQuNTE2IDgzOC4wNTUgNzE4Ljg2NyA4MzkuODYxQzcxMy4yMTggODQxLjY2NyA3 + MDguMzk3IDg0NC4xMzIgNzA0LjQwMyA4NDcuMjU2TDY5My4wNjIgODI1LjE1MkM2OTkuMDg1IDgy + MC44MzQgNzA2LjI5NiA4MTcuNTQ4IDcxNC42OTQgODE1LjI5NUM3MjMuMDkzIDgxMy4wNDEgNzMx + LjYzNiA4MTEuOTE0IDc0MC4zMjMgODExLjkxNEM3NTcuMDg3IDgxMS45MTQgNzcwLjA3MyA4MTUu + ODQyIDc3OS4yODIgODIzLjY5NkM3ODguNDkgODMxLjU1MSA3OTMuMDk1IDg0My43OTMgNzkzLjA5 + NSA4NjAuNDIzTDc5My4wOTUgOTIyLjk4N0w3NjMuMjM4IDkyMi45ODdaTTcyOS45NTggOTI0LjUz + M0M3MjEuNDM5IDkyNC41MzMgNzE0LjEyNSA5MjMuMDk1IDcwOC4wMTcgOTIwLjIxOEM3MDEuOTA5 + IDkxNy4zNDEgNjk3LjIzMiA5MTMuMzkyIDY5My45ODcgOTA4LjM3QzY5MC43NDIgOTAzLjM0OSA2 + ODkuMTIgODk3LjY5IDY4OS4xMiA4OTEuMzkzQzY4OS4xMiA4ODQuOTM3IDY5MC43MDYgODc5LjI1 + OSA2OTMuODc5IDg3NC4zNTlDNjk3LjA1MiA4NjkuNDU5IDcwMi4wOTggODY1LjYxNyA3MDkuMDE3 + IDg2Mi44MzNDNzE1LjkzNyA4NjAuMDQ5IDcyNC45NjEgODU4LjY1OCA3MzYuMDg5IDg1OC42NThM + NzY1LjA3NCA4NTguNjU4TDc2NS4wNzQgODc3LjE2Nkw3MzkuNjExIDg3Ny4xNjZDNzMyLjEwMyA4 + NzcuMTY2IDcyNi45NjMgODc4LjM3NyA3MjQuMTkxIDg4MC43OThDNzIxLjQxOSA4ODMuMjE5IDcy + MC4wMzMgODg2LjMxOCA3MjAuMDMzIDg5MC4wOTVDNzIwLjAzMyA4OTQuMDY2IDcyMS42MTEgODk3 + LjI0NyA3MjQuNzY3IDg5OS42NDFDNzI3LjkyMiA5MDIuMDM0IDczMi4yNjIgOTAzLjIzMSA3Mzcu + Nzg0IDkwMy4yMzFDNzQzLjExOCA5MDMuMjMxIDc0Ny45MDcgOTAxLjk5MyA3NTIuMTUxIDg5OS41 + MThDNzU2LjM5NSA4OTcuMDQzIDc1OS40NTYgODkzLjMxOCA3NjEuMzMzIDg4OC4zNDJMNzY2LjEz + IDkwMy4wOTFDNzYzLjg3NSA5MTAuMDYgNzU5LjY5MyA5MTUuMzc2IDc1My41ODEgOTE5LjAzOUM3 + NDcuNDcgOTIyLjcwMiA3MzkuNTk2IDkyNC41MzMgNzI5Ljk1OCA5MjQuNTMzWiIgZmlsbD0iIzk5 + OTk5OSIgZmlsbC1ydWxlPSJub256ZXJvIiBvcGFjaXR5PSIxIiBzdHJva2U9Im5vbmUiLz4KPHBh + dGggZD0iTTg4OC4yMDcgODExLjkxNEM4OTYuOTIyIDgxMS45MTQgOTA0LjY5OSA4MTMuNjI0IDkx + MS41NCA4MTcuMDQ0QzkxOC4zODEgODIwLjQ2NCA5MjMuNzgzIDgyNS43MzcgOTI3Ljc0NiA4MzIu + ODYyQzkzMS43MDkgODM5Ljk4NyA5MzMuNjkgODQ5LjE0IDkzMy42OSA4NjAuMzIxTDkzMy42OSA5 + MjIuOTg3TDkwMS45MjggOTIyLjk4N0w5MDEuOTI4IDg2NS4wNjhDOTAxLjkyOCA4NTYuMjYyIDg5 + OS45OSA4NDkuNzY1IDg5Ni4xMTMgODQ1LjU3OEM4OTIuMjM2IDg0MS4zOSA4ODYuODAxIDgzOS4y + OTYgODc5LjgwOCA4MzkuMjk2Qzg3NC43MiA4MzkuMjk2IDg3MC4xNzcgODQwLjM1NiA4NjYuMTgg + ODQyLjQ3N0M4NjIuMTgyIDg0NC41OTggODU5LjA3NSA4NDcuODIgODU2Ljg1OCA4NTIuMTQyQzg1 + NC42NDIgODU2LjQ2NSA4NTMuNTM0IDg2Mi4wMjMgODUzLjUzNCA4NjguODE5TDg1My41MzQgOTIy + Ljk4N0w4MjEuNzcyIDkyMi45ODdMODIxLjc3MiA4MTMuNDU3TDg1Mi4wMzIgODEzLjQ1N0w4NTIu + MDMyIDg0My44MjNMODQ2LjM0NyA4MzQuNzAyQzg1MC4yODEgODI3LjI3MSA4NTUuOTA3IDgyMS42 + MTUgODYzLjIyMyA4MTcuNzM1Qzg3MC41MzkgODEzLjg1NSA4NzguODY3IDgxMS45MTQgODg4LjIw + NyA4MTEuOTE0WiIgZmlsbD0iIzk5OTk5OSIgZmlsbC1ydWxlPSJub256ZXJvIiBvcGFjaXR5PSIx + IiBzdHJva2U9Im5vbmUiLz4KPC9nPgo8L2c+Cjwvc3ZnPgo= + mediatype: image/svg+xml install: spec: clusterPermissions: - - rules: - - apiGroups: - - apps - resources: - - daemonsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - - apiGroups: - - bpfman.io - resources: - - bpfprograms - verbs: - - get - - list - - watch - - apiGroups: - - bpfman.io - resources: - - configmaps/finalizers - verbs: - - update - - apiGroups: - - bpfman.io - resources: - - fentryprograms - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - - apiGroups: - - bpfman.io - resources: - - fentryprograms/finalizers - verbs: - - update - - apiGroups: - - bpfman.io - resources: - - fentryprograms/status - verbs: - - get - - patch - - update - - apiGroups: - - bpfman.io - resources: - - fexitprograms - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - - apiGroups: - - bpfman.io - resources: - - fexitprograms/finalizers - verbs: - - update - - apiGroups: - - bpfman.io - resources: - - fexitprograms/status - verbs: - - get - - patch - - update - - apiGroups: - - bpfman.io - resources: - - kprobeprograms - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - - apiGroups: - - bpfman.io - resources: - - kprobeprograms/finalizers - verbs: - - update - - apiGroups: - - bpfman.io - resources: - - kprobeprograms/status - verbs: - - get - - patch - - update - - apiGroups: - - bpfman.io - resources: - - tcprograms - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - - apiGroups: - - bpfman.io - resources: - - tcprograms/finalizers - verbs: - - update - - apiGroups: - - bpfman.io - resources: - - tcprograms/status - verbs: - - get - - patch - - update - - apiGroups: - - bpfman.io - resources: - - tracepointprograms - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - - apiGroups: - - bpfman.io - resources: - - tracepointprograms/finalizers - verbs: - - update - - apiGroups: - - bpfman.io - resources: - - tracepointprograms/status - verbs: - - get - - patch - - update - - apiGroups: - - bpfman.io - resources: - - uprobeprograms - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - - apiGroups: - - bpfman.io - resources: - - uprobeprograms/finalizers - verbs: - - update - - apiGroups: - - bpfman.io - resources: - - uprobeprograms/status - verbs: - - get - - patch - - update - - apiGroups: - - bpfman.io - resources: - - xdpprograms - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - - apiGroups: - - bpfman.io - resources: - - xdpprograms/finalizers - verbs: - - update - - apiGroups: - - bpfman.io - resources: - - xdpprograms/status - verbs: - - get - - patch - - update - - apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - get - - list - - watch - - apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch - - apiGroups: - - security.openshift.io - resources: - - securitycontextconstraints - verbs: - - create - - delete - - get - - list - - watch - - apiGroups: - - storage.k8s.io - resources: - - csidrivers - verbs: - - create - - delete - - get - - list - - watch - - apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create - - apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create - serviceAccountName: bpfman-operator + - rules: + - apiGroups: + - apps + resources: + - daemonsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - bpfman.io + resources: + - bpfapplications + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - bpfman.io + resources: + - bpfapplications/finalizers + verbs: + - update + - apiGroups: + - bpfman.io + resources: + - bpfapplications/status + verbs: + - get + - patch + - update + - apiGroups: + - bpfman.io + resources: + - bpfprograms + verbs: + - get + - list + - watch + - apiGroups: + - bpfman.io + resources: + - configmaps/finalizers + verbs: + - update + - apiGroups: + - bpfman.io + resources: + - fentryprograms + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - bpfman.io + resources: + - fentryprograms/finalizers + verbs: + - update + - apiGroups: + - bpfman.io + resources: + - fentryprograms/status + verbs: + - get + - patch + - update + - apiGroups: + - bpfman.io + resources: + - fexitprograms + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - bpfman.io + resources: + - fexitprograms/finalizers + verbs: + - update + - apiGroups: + - bpfman.io + resources: + - fexitprograms/status + verbs: + - get + - patch + - update + - apiGroups: + - bpfman.io + resources: + - kprobeprograms + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - bpfman.io + resources: + - kprobeprograms/finalizers + verbs: + - update + - apiGroups: + - bpfman.io + resources: + - kprobeprograms/status + verbs: + - get + - patch + - update + - apiGroups: + - bpfman.io + resources: + - tcprograms + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - bpfman.io + resources: + - tcprograms/finalizers + verbs: + - update + - apiGroups: + - bpfman.io + resources: + - tcprograms/status + verbs: + - get + - patch + - update + - apiGroups: + - bpfman.io + resources: + - tracepointprograms + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - bpfman.io + resources: + - tracepointprograms/finalizers + verbs: + - update + - apiGroups: + - bpfman.io + resources: + - tracepointprograms/status + verbs: + - get + - patch + - update + - apiGroups: + - bpfman.io + resources: + - uprobeprograms + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - bpfman.io + resources: + - uprobeprograms/finalizers + verbs: + - update + - apiGroups: + - bpfman.io + resources: + - uprobeprograms/status + verbs: + - get + - patch + - update + - apiGroups: + - bpfman.io + resources: + - xdpprograms + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - bpfman.io + resources: + - xdpprograms/finalizers + verbs: + - update + - apiGroups: + - bpfman.io + resources: + - xdpprograms/status + verbs: + - get + - patch + - update + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - get + - list + - watch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + verbs: + - create + - delete + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - csidrivers + verbs: + - create + - delete + - get + - list + - watch + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create + serviceAccountName: bpfman-operator deployments: - - label: - app.kubernetes.io/component: manager - app.kubernetes.io/created-by: bpfman-operator - app.kubernetes.io/instance: controller-manager - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: deployment - app.kubernetes.io/part-of: bpfman-operator - control-plane: controller-manager - name: bpfman-operator - spec: - replicas: 1 - selector: - matchLabels: + - label: + app.kubernetes.io/component: manager + app.kubernetes.io/created-by: bpfman-operator + app.kubernetes.io/instance: controller-manager + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: deployment + app.kubernetes.io/part-of: bpfman-operator + control-plane: controller-manager + name: bpfman-operator + spec: + replicas: 1 + selector: + matchLabels: + control-plane: controller-manager + strategy: {} + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + labels: control-plane: controller-manager - strategy: {} - template: - metadata: - annotations: - kubectl.kubernetes.io/default-container: manager - labels: - control-plane: controller-manager - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/arch - operator: In - values: - - amd64 - - arm64 - - ppc64le - - s390x - - key: kubernetes.io/os - operator: In - values: - - linux - containers: - - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8174/ - - --logtostderr=true - - --v=0 - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.0 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - - args: - - --health-probe-bind-address=:8175 - - --metrics-bind-address=127.0.0.1:8174 - - --leader-elect - command: - - /bpfman-operator - env: - - name: GO_LOG - value: debug - image: quay.io/bpfman/bpfman-operator:latest - imagePullPolicy: IfNotPresent - livenessProbe: - httpGet: - path: /healthz - port: 8175 - initialDelaySeconds: 15 - periodSeconds: 20 - name: bpfman-operator - readinessProbe: - httpGet: - path: /readyz - port: 8175 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 10m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - arm64 + - ppc64le + - s390x + - key: kubernetes.io/os + operator: In + values: + - linux + containers: + - args: + - --secure-listen-address=0.0.0.0:8443 + - --upstream=http://127.0.0.1:8174/ + - --logtostderr=true + - --v=0 + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.0 + name: kube-rbac-proxy + ports: + - containerPort: 8443 + name: https + protocol: TCP + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 5m + memory: 64Mi securityContext: - runAsNonRoot: true - serviceAccountName: bpfman-operator - terminationGracePeriodSeconds: 10 + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + - args: + - --health-probe-bind-address=:8175 + - --metrics-bind-address=127.0.0.1:8174 + - --leader-elect + command: + - /bpfman-operator + env: + - name: GO_LOG + value: debug + image: quay.io/bpfman/bpfman-operator:latest + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /healthz + port: 8175 + initialDelaySeconds: 15 + periodSeconds: 20 + name: bpfman-operator + readinessProbe: + httpGet: + path: /readyz + port: 8175 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 10m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + securityContext: + runAsNonRoot: true + serviceAccountName: bpfman-operator + terminationGracePeriodSeconds: 10 permissions: - - rules: - - apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - serviceAccountName: bpfman-operator + - rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + serviceAccountName: bpfman-operator strategy: deployment installModes: - - supported: false - type: OwnNamespace - - supported: false - type: SingleNamespace - - supported: false - type: MultiNamespace - - supported: true - type: AllNamespaces + - supported: false + type: OwnNamespace + - supported: false + type: SingleNamespace + - supported: false + type: MultiNamespace + - supported: true + type: AllNamespaces keywords: - - ebpf - - kubernetes + - ebpf + - kubernetes links: - - name: bpfman website - url: https://bpfman.io/ + - name: bpfman website + url: https://bpfman.io/ maintainers: - - email: astoycos@redhat.com - name: Andrew Stoycos + - email: astoycos@redhat.com + name: Andrew Stoycos maturity: alpha provider: name: The bpfman Community diff --git a/config/samples/bpfman.io_v1alpha1_bpfapplication.yaml b/config/samples/bpfman.io_v1alpha1_bpfapplication.yaml index 18e1a0c9e..bce82eac0 100644 --- a/config/samples/bpfman.io_v1alpha1_bpfapplication.yaml +++ b/config/samples/bpfman.io_v1alpha1_bpfapplication.yaml @@ -21,7 +21,7 @@ spec: tracepoint: bpffunctionname: tracepoint_kill_recorder names: - - syscalls/sys_enter_kill + - syscalls/sys_enter_kill - type: TC tc: bpffunctionname: stats @@ -39,7 +39,7 @@ spec: namespace: bpfman pods: matchLabels: - name: bpfman-daemon + name: bpfman-daemon containernames: - bpfman - bpfman-agent @@ -48,4 +48,4 @@ spec: bpffunctionname: xdp_stats interfaceselector: primarynodeinterface: true - priority: 55 \ No newline at end of file + priority: 55 diff --git a/controllers/bpfman-agent/application-program.go b/controllers/bpfman-agent/application-program.go index 554eef07c..32a3f6a2e 100644 --- a/controllers/bpfman-agent/application-program.go +++ b/controllers/bpfman-agent/application-program.go @@ -3,6 +3,7 @@ package bpfmanagent import ( "context" "fmt" + "strings" bpfmaniov1alpha1 "github.com/bpfman/bpfman-operator/apis/v1alpha1" "github.com/bpfman/bpfman-operator/internal" @@ -66,13 +67,19 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque var err error var complete bool + namePrefix := func( + app bpfmaniov1alpha1.BpfApplication, + prog bpfmaniov1alpha1.BpfApplicationProgram) string { + return app.Name + "-" + strings.ToLower(string(prog.Type)) + "-" + } + for i, a := range appPrograms.Items { for j, p := range a.Spec.Programs { switch p.Type { case bpfmaniov1alpha1.ProgTypeFentry: fentryProgram := bpfmaniov1alpha1.FentryProgram{ ObjectMeta: metav1.ObjectMeta{ - Name: a.Name + "-fentry", + Name: namePrefix(a, p) + sanitize(p.Fentry.FunctionName), }, Spec: bpfmaniov1alpha1.FentryProgramSpec{ FentryProgramInfo: *p.Fentry, @@ -92,7 +99,7 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque case bpfmaniov1alpha1.ProgTypeFexit: fexitProgram := bpfmaniov1alpha1.FexitProgram{ ObjectMeta: metav1.ObjectMeta{ - Name: a.Name + "-fexit", + Name: namePrefix(a, p) + sanitize(p.Fexit.FunctionName), }, Spec: bpfmaniov1alpha1.FexitProgramSpec{ FexitProgramInfo: *p.Fexit, @@ -113,7 +120,7 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque bpfmaniov1alpha1.ProgTypeKretprobe: kprobeProgram := bpfmaniov1alpha1.KprobeProgram{ ObjectMeta: metav1.ObjectMeta{ - Name: a.Name + "-kprobe", + Name: namePrefix(a, p) + sanitize(p.Kprobe.FunctionName), }, Spec: bpfmaniov1alpha1.KprobeProgramSpec{ KprobeProgramInfo: *p.Kprobe, @@ -134,7 +141,7 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque bpfmaniov1alpha1.ProgTypeUretprobe: uprobeProgram := bpfmaniov1alpha1.UprobeProgram{ ObjectMeta: metav1.ObjectMeta{ - Name: a.Name + "-uprobe", + Name: namePrefix(a, p) + sanitize(p.Uprobe.FunctionName), }, Spec: bpfmaniov1alpha1.UprobeProgramSpec{ UprobeProgramInfo: *p.Uprobe, @@ -154,7 +161,7 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque case bpfmaniov1alpha1.ProgTypeTracepoint: tracepointProgram := bpfmaniov1alpha1.TracepointProgram{ ObjectMeta: metav1.ObjectMeta{ - Name: a.Name + "-tracepoint", + Name: namePrefix(a, p) + sanitize(p.Tracepoint.Names[0]), }, Spec: bpfmaniov1alpha1.TracepointProgramSpec{ TracepointProgramInfo: *p.Tracepoint, @@ -173,9 +180,15 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque case bpfmaniov1alpha1.ProgTypeTC, bpfmaniov1alpha1.ProgTypeTCX: + interfaces, ifErr := getInterfaces(&p.TC.InterfaceSelector, r.ourNode) + if ifErr != nil { + ctxLogger.Error(ifErr, "failed to get interfaces for TC Program", + "app program name", a.Name, "program index", j) + continue + } tcProgram := bpfmaniov1alpha1.TcProgram{ ObjectMeta: metav1.ObjectMeta{ - Name: a.Name + "-tc", + Name: namePrefix(a, p) + p.TC.Direction + "-" + interfaces[0], }, Spec: bpfmaniov1alpha1.TcProgramSpec{ TcProgramInfo: *p.TC, @@ -193,9 +206,15 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque complete, res, err = r.reconcileCommon(ctx, rec, tcObjects) case bpfmaniov1alpha1.ProgTypeXDP: + interfaces, ifErr := getInterfaces(&p.XDP.InterfaceSelector, r.ourNode) + if ifErr != nil { + ctxLogger.Error(ifErr, "failed to get interfaces for XDP Program", + "app program name", a.Name, "program index", j) + continue + } xdpProgram := bpfmaniov1alpha1.XdpProgram{ ObjectMeta: metav1.ObjectMeta{ - Name: a.Name + "-xdp", + Name: namePrefix(a, p) + interfaces[0], }, Spec: bpfmaniov1alpha1.XdpProgramSpec{ XdpProgramInfo: *p.XDP, diff --git a/controllers/bpfman-agent/application-program_test.go b/controllers/bpfman-agent/application-program_test.go index f6d668d10..9b5f1d211 100644 --- a/controllers/bpfman-agent/application-program_test.go +++ b/controllers/bpfman-agent/application-program_test.go @@ -36,13 +36,13 @@ func TestBpfApplicationControllerCreate(t *testing.T) { // fentry program config bpfFentryFunctionName = "fentry_test" fentryFunctionName = "do_unlinkat" - fentryBpfProgName = fmt.Sprintf("%s-%s-%s-%s", name, "fentry", fakeNode.Name, "do-unlinkat") + fentryBpfProgName = fmt.Sprintf("%s-%s-%s-%s-%s", name, "fentry", "do-unlinkat", fakeNode.Name, "do-unlinkat") fentryBpfProg = &bpfmaniov1alpha1.BpfProgram{} fentryFakeUID = "ef71d42c-aa21-48e8-a697-82391d801a81" // kprobe program config bpfKprobeFunctionName = "kprobe_test" kprobeFunctionName = "try_to_wake_up" - kprobeBpfProgName = fmt.Sprintf("%s-%s-%s-%s", name, "kprobe", fakeNode.Name, "try-to-wake-up") + kprobeBpfProgName = fmt.Sprintf("%s-%s-%s-%s-%s", name, "kprobe", "try-to-wake-up", fakeNode.Name, "try-to-wake-up") kprobeBpfProg = &bpfmaniov1alpha1.BpfProgram{} kprobeFakeUID = "ef71d42c-aa21-48e8-a697-82391d801a82" kprobeOffset = 0 diff --git a/controllers/bpfman-agent/common.go b/controllers/bpfman-agent/common.go index ba2672ae5..0f0a50937 100644 --- a/controllers/bpfman-agent/common.go +++ b/controllers/bpfman-agent/common.go @@ -21,6 +21,7 @@ import ( "fmt" "reflect" "strconv" + "strings" "time" v1 "k8s.io/api/core/v1" @@ -891,3 +892,9 @@ func getClientset() (*kubernetes.Clientset, error) { return clientset, nil } + +// sanitize a string to work as a bpfProgram name +func sanitize(name string) string { + name = strings.TrimPrefix(name, "/") + return strings.Replace(strings.Replace(name, "/", "-", -1), "_", "-", -1) +} diff --git a/controllers/bpfman-agent/fentry-program.go b/controllers/bpfman-agent/fentry-program.go index f402d5cad..69a8d70ee 100644 --- a/controllers/bpfman-agent/fentry-program.go +++ b/controllers/bpfman-agent/fentry-program.go @@ -19,7 +19,6 @@ package bpfmanagent import ( "context" "fmt" - "strings" bpfmaniov1alpha1 "github.com/bpfman/bpfman-operator/apis/v1alpha1" bpfmanagentinternal "github.com/bpfman/bpfman-operator/controllers/bpfman-agent/internal" @@ -124,8 +123,7 @@ func (r *FentryProgramReconciler) SetupWithManager(mgr ctrl.Manager) error { func (r *FentryProgramReconciler) getExpectedBpfPrograms(ctx context.Context) (*bpfmaniov1alpha1.BpfProgramList, error) { progs := &bpfmaniov1alpha1.BpfProgramList{} - // sanitize fentry name to work in a bpfProgram name - sanatizedFentry := strings.Replace(strings.Replace(r.currentFentryProgram.Spec.FunctionName, "/", "-", -1), "_", "-", -1) + sanatizedFentry := sanitize(r.currentFentryProgram.Spec.FunctionName) bpfProgramName := fmt.Sprintf("%s-%s-%s", r.currentFentryProgram.Name, r.NodeName, sanatizedFentry) annotations := map[string]string{internal.FentryProgramFunction: r.currentFentryProgram.Spec.FunctionName} diff --git a/controllers/bpfman-agent/fexit-program.go b/controllers/bpfman-agent/fexit-program.go index 7e92f6e8f..6741a71bb 100644 --- a/controllers/bpfman-agent/fexit-program.go +++ b/controllers/bpfman-agent/fexit-program.go @@ -19,7 +19,6 @@ package bpfmanagent import ( "context" "fmt" - "strings" bpfmaniov1alpha1 "github.com/bpfman/bpfman-operator/apis/v1alpha1" bpfmanagentinternal "github.com/bpfman/bpfman-operator/controllers/bpfman-agent/internal" @@ -124,8 +123,7 @@ func (r *FexitProgramReconciler) SetupWithManager(mgr ctrl.Manager) error { func (r *FexitProgramReconciler) getExpectedBpfPrograms(ctx context.Context) (*bpfmaniov1alpha1.BpfProgramList, error) { progs := &bpfmaniov1alpha1.BpfProgramList{} - // sanitize fexit name to work in a bpfProgram name - sanatizedFexit := strings.Replace(strings.Replace(r.currentFexitProgram.Spec.FunctionName, "/", "-", -1), "_", "-", -1) + sanatizedFexit := sanitize(r.currentFexitProgram.Spec.FunctionName) bpfProgramName := fmt.Sprintf("%s-%s-%s", r.currentFexitProgram.Name, r.NodeName, sanatizedFexit) annotations := map[string]string{internal.FexitProgramFunction: r.currentFexitProgram.Spec.FunctionName} diff --git a/controllers/bpfman-agent/kprobe-program.go b/controllers/bpfman-agent/kprobe-program.go index 7e79dc7e8..eaeb684fb 100644 --- a/controllers/bpfman-agent/kprobe-program.go +++ b/controllers/bpfman-agent/kprobe-program.go @@ -19,7 +19,6 @@ package bpfmanagent import ( "context" "fmt" - "strings" bpfmaniov1alpha1 "github.com/bpfman/bpfman-operator/apis/v1alpha1" bpfmanagentinternal "github.com/bpfman/bpfman-operator/controllers/bpfman-agent/internal" @@ -124,8 +123,7 @@ func (r *KprobeProgramReconciler) SetupWithManager(mgr ctrl.Manager) error { func (r *KprobeProgramReconciler) getExpectedBpfPrograms(ctx context.Context) (*bpfmaniov1alpha1.BpfProgramList, error) { progs := &bpfmaniov1alpha1.BpfProgramList{} - // sanitize kprobe name to work in a bpfProgram name - sanatizedKprobe := strings.Replace(strings.Replace(r.currentKprobeProgram.Spec.FunctionName, "/", "-", -1), "_", "-", -1) + sanatizedKprobe := sanitize(r.currentKprobeProgram.Spec.FunctionName) bpfProgramName := fmt.Sprintf("%s-%s-%s", r.currentKprobeProgram.Name, r.NodeName, sanatizedKprobe) annotations := map[string]string{internal.KprobeProgramFunction: r.currentKprobeProgram.Spec.FunctionName} diff --git a/controllers/bpfman-agent/tracepoint-program.go b/controllers/bpfman-agent/tracepoint-program.go index 5d701d07d..b7350fe2d 100644 --- a/controllers/bpfman-agent/tracepoint-program.go +++ b/controllers/bpfman-agent/tracepoint-program.go @@ -19,7 +19,6 @@ package bpfmanagent import ( "context" "fmt" - "strings" bpfmaniov1alpha1 "github.com/bpfman/bpfman-operator/apis/v1alpha1" bpfmanagentinternal "github.com/bpfman/bpfman-operator/controllers/bpfman-agent/internal" @@ -125,8 +124,7 @@ func (r *TracepointProgramReconciler) getExpectedBpfPrograms(ctx context.Context progs := &bpfmaniov1alpha1.BpfProgramList{} for _, tracepoint := range r.currentTracepointProgram.Spec.Names { - // sanitize tracepoint name to work in a bpfProgram name - sanatizedTrace := strings.Replace(strings.Replace(tracepoint, "/", "-", -1), "_", "-", -1) + sanatizedTrace := sanitize(tracepoint) bpfProgramName := fmt.Sprintf("%s-%s-%s", r.currentTracepointProgram.Name, r.NodeName, sanatizedTrace) annotations := map[string]string{internal.TracepointProgramTracepoint: tracepoint} diff --git a/controllers/bpfman-agent/uprobe-program.go b/controllers/bpfman-agent/uprobe-program.go index ebe7e4929..8db4b2b00 100644 --- a/controllers/bpfman-agent/uprobe-program.go +++ b/controllers/bpfman-agent/uprobe-program.go @@ -20,7 +20,6 @@ import ( "context" "fmt" "strconv" - "strings" bpfmaniov1alpha1 "github.com/bpfman/bpfman-operator/apis/v1alpha1" bpfmanagentinternal "github.com/bpfman/bpfman-operator/controllers/bpfman-agent/internal" @@ -157,8 +156,7 @@ func (r *UprobeProgramReconciler) getUprobeContainerInfo(ctx context.Context) (* func (r *UprobeProgramReconciler) getExpectedBpfPrograms(ctx context.Context) (*bpfmaniov1alpha1.BpfProgramList, error) { progs := &bpfmaniov1alpha1.BpfProgramList{} - // sanitize uprobe name to work in a bpfProgram name - sanatizedUprobe := strings.Replace(strings.Replace(r.currentUprobeProgram.Spec.Target, "/", "-", -1), "_", "-", -1) + sanatizedUprobe := sanitize(r.currentUprobeProgram.Spec.Target) bpfProgramNameBase := fmt.Sprintf("%s-%s-%s", r.currentUprobeProgram.Name, r.NodeName, sanatizedUprobe) if r.currentUprobeProgram.Spec.Containers != nil {