From dd6548c4a26031324975d97aee345e4e6a2e7efa Mon Sep 17 00:00:00 2001 From: Leonardo Cecchi Date: Tue, 1 Oct 2024 15:40:48 +0200 Subject: [PATCH] feat: operator plugin and manifests (#18) Signed-off-by: Leonardo Cecchi --- cmd/operator/main.go | 231 ++++++------------ config/manager/manager.yaml | 2 +- config/rbac/leader_election_role_binding.yaml | 4 +- config/rbac/metrics_auth_role_binding.yaml | 4 +- config/rbac/role_binding.yaml | 4 +- config/rbac/service_account.yaml | 3 +- docs/examples/cluster-example.yaml | 12 + go.mod | 9 +- go.sum | 4 + internal/cnpgi/instance/identity.go | 3 +- .../cnpgi/{instance => metadata}/constants.go | 2 +- internal/cnpgi/metadata/doc.go | 3 + internal/cnpgi/operator/constants.go | 18 -- internal/cnpgi/operator/identity.go | 4 +- internal/cnpgi/operator/lifecycle.go | 79 ++++++ internal/cnpgi/operator/start.go | 21 +- internal/manager/manager.go | 161 ++++++++++++ kubernetes/certificate-issuer.yaml | 6 + kubernetes/client-certificate.yaml | 19 ++ kubernetes/deployment.yaml | 50 ++++ kubernetes/kustomization.yaml | 17 ++ kubernetes/server-certificate.yaml | 21 ++ kubernetes/service.yaml | 18 ++ scripts/run.sh | 22 ++ 24 files changed, 511 insertions(+), 206 deletions(-) create mode 100644 docs/examples/cluster-example.yaml rename internal/cnpgi/{instance => metadata}/constants.go (97%) create mode 100644 internal/cnpgi/metadata/doc.go delete mode 100644 internal/cnpgi/operator/constants.go create mode 100644 internal/cnpgi/operator/lifecycle.go create mode 100644 internal/manager/manager.go create mode 100644 kubernetes/certificate-issuer.yaml create mode 100644 kubernetes/client-certificate.yaml create mode 100644 kubernetes/deployment.yaml create mode 100644 kubernetes/kustomization.yaml create mode 100644 kubernetes/server-certificate.yaml create mode 100644 kubernetes/service.yaml create mode 100755 scripts/run.sh diff --git a/cmd/operator/main.go b/cmd/operator/main.go index 202cd16..f0954e8 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -1,177 +1,90 @@ -/* -Copyright 2024. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package main contains the implementation of the CNPG-i operator plugin +// Package main is the entrypoint of operator plugin package main import ( - "crypto/tls" - "flag" + "context" + "fmt" "os" - // +kubebuilder:scaffold:imports - "k8s.io/apimachinery/pkg/runtime" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "github.com/cloudnative-pg/machinery/pkg/log" + "github.com/sourcegraph/conc/pool" + "github.com/spf13/cobra" + "github.com/spf13/viper" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/healthz" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - "sigs.k8s.io/controller-runtime/pkg/metrics/filters" - metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" - "sigs.k8s.io/controller-runtime/pkg/webhook" - barmancloudv1 "github.com/cloudnative-pg/plugin-barman-cloud/api/v1" "github.com/cloudnative-pg/plugin-barman-cloud/internal/cnpgi/operator" - "github.com/cloudnative-pg/plugin-barman-cloud/internal/controller" - - // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) - // to ensure that exec-entrypoint and run can make use of them. - _ "k8s.io/client-go/plugin/pkg/client/auth" + "github.com/cloudnative-pg/plugin-barman-cloud/internal/manager" ) -var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") -) - -func init() { - utilruntime.Must(clientgoscheme.AddToScheme(scheme)) - - utilruntime.Must(barmancloudv1.AddToScheme(scheme)) - // +kubebuilder:scaffold:scheme -} - func main() { - var metricsAddr string - var enableLeaderElection bool - var probeAddr string - var secureMetrics bool - var enableHTTP2 bool - var tlsOpts []func(*tls.Config) - flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ - "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") - flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") - flag.BoolVar(&enableLeaderElection, "leader-elect", false, - "Enable leader election for controller manager. "+ - "Enabling this will ensure there is only one active controller manager.") - flag.BoolVar(&secureMetrics, "metrics-secure", true, - "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") - flag.BoolVar(&enableHTTP2, "enable-http2", false, - "If set, HTTP/2 will be enabled for the metrics and webhook servers") - opts := zap.Options{ - Development: true, - } - opts.BindFlags(flag.CommandLine) - flag.Parse() + cobra.EnableTraverseRunHooks = true - ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) - - // if the enable-http2 flag is false (the default), http/2 should be disabled - // due to its vulnerabilities. More specifically, disabling http/2 will - // prevent from being vulnerable to the HTTP/2 Stream Cancellation and - // Rapid Reset CVEs. For more information see: - // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 - // - https://github.com/advisories/GHSA-4374-p667-p6c8 - disableHTTP2 := func(c *tls.Config) { - setupLog.Info("disabling http/2") - c.NextProtos = []string{"http/1.1"} + logFlags := &log.Flags{} + rootCmd := &cobra.Command{ + Use: "plugin-barman-cloud", + PersistentPreRunE: func(_ *cobra.Command, _ []string) error { + logFlags.ConfigureLogging() + return nil + }, } - if !enableHTTP2 { - tlsOpts = append(tlsOpts, disableHTTP2) - } + logFlags.AddFlags(rootCmd.PersistentFlags()) + rootCmd.AddCommand(newOperatorCommand()) - webhookServer := webhook.NewServer(webhook.Options{ - TLSOpts: tlsOpts, - }) - - // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. - // More info: - // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.0/pkg/metrics/server - // - https://book.kubebuilder.io/reference/metrics.html - metricsServerOptions := metricsserver.Options{ - BindAddress: metricsAddr, - SecureServing: secureMetrics, - // TODO(user): TLSOpts is used to allow configuring the TLS config used for the server. If certificates are - // not provided, self-signed certificates will be generated by default. This option is not recommended for - // production environments as self-signed certificates do not offer the same level of trust and security - // as certificates issued by a trusted Certificate Authority (CA). The primary risk is potentially allowing - // unauthorized access to sensitive metrics data. Consider replacing with CertDir, CertName, and KeyName - // to provide certificates, ensuring the server communicates using trusted and secure certificates. - TLSOpts: tlsOpts, - } - - if secureMetrics { - // FilterProvider is used to protect the metrics endpoint with authn/authz. - // These configurations ensure that only authorized users and service accounts - // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: - // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.0/pkg/metrics/filters#WithAuthenticationAndAuthorization - metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - } - - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - Metrics: metricsServerOptions, - WebhookServer: webhookServer, - HealthProbeBindAddress: probeAddr, - LeaderElection: enableLeaderElection, - LeaderElectionID: "822e3f5c.cnpg.io", - // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily - // when the Manager ends. This requires the binary to immediately end when the - // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly - // speeds up voluntary leader transitions as the new leader don't have to wait - // LeaseDuration time first. - // - // In the default scaffold provided, the program ends immediately after - // the manager stops, so would be fine to enable this option. However, - // if you are doing or is intended to do any operation such as perform cleanups - // after the manager stops then its usage might be unsafe. - // LeaderElectionReleaseOnCancel: true, - }) - if err != nil { - setupLog.Error(err, "unable to start manager") - os.Exit(1) - } - - if err = (&controller.ObjectStoreReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "ObjectStore") - os.Exit(1) - } - // +kubebuilder:scaffold:builder - - if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up health check") - os.Exit(1) - } - if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up ready check") - os.Exit(1) - } - - if err := mgr.Add(&operator.CNPGI{}); err != nil { - setupLog.Error(err, "unable to create CNPGI webserver") - os.Exit(1) - } - - setupLog.Info("starting manager") - if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { - setupLog.Error(err, "problem running manager") + if err := rootCmd.Execute(); err != nil { + fmt.Println(err) os.Exit(1) } } + +func newOperatorCommand() *cobra.Command { + cmd := operator.NewCommand() + cmd.Use = "operator" + cmd.Short = "Starts the BarmanObjectStore reconciler and the Barman Cloud CNPG-i plugin" + grpcServer := cmd.RunE + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + ctrl.SetupSignalHandler() + operatorPool := pool. + New(). + WithContext(cmd.Context()). + WithCancelOnError(). + WithFirstError() + operatorPool.Go(func(ctx context.Context) error { + cmd.SetContext(ctx) + + if len(viper.GetString("sidecar-image")) == 0 { + return fmt.Errorf("missing required SIDECAR_IMAGE environment variable") + } + + err := grpcServer(cmd, args) + return err + }) + operatorPool.Go(manager.Start) + return operatorPool.Wait() + } + + cmd.Flags().String("metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ + "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") + _ = viper.BindPFlag("metrics-bind-address", cmd.Flags().Lookup("metrics-bind-address")) + + cmd.Flags().String("health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + _ = viper.BindPFlag("health-probe-bind-address", cmd.Flags().Lookup("health-probe-bind-address")) + + cmd.Flags().Bool("leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + _ = viper.BindPFlag("leader-elect", cmd.Flags().Lookup("leader-elect")) + + cmd.Flags().Bool("metrics-secure", true, + "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + _ = viper.BindPFlag("metrics-secure", cmd.Flags().Lookup("metrics-secure")) + + cmd.Flags().Bool("enable-http2", false, + "If set, HTTP/2 will be enabled for the metrics and webhook servers") + _ = viper.BindPFlag("enable-http2", cmd.Flags().Lookup("enable-http2")) + + _ = viper.BindEnv("sidecar-image", "SIDECAR_IMAGE") + + return cmd +} diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 954628f..eb47c31 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -91,5 +91,5 @@ spec: requests: cpu: 10m memory: 64Mi - serviceAccountName: controller-manager + serviceAccountName: plugin-barman-cloud terminationGracePeriodSeconds: 10 diff --git a/config/rbac/leader_election_role_binding.yaml b/config/rbac/leader_election_role_binding.yaml index 0829670..5915f2c 100644 --- a/config/rbac/leader_election_role_binding.yaml +++ b/config/rbac/leader_election_role_binding.yaml @@ -11,5 +11,5 @@ roleRef: name: leader-election-role subjects: - kind: ServiceAccount - name: controller-manager - namespace: system + name: plugin-barman-cloud + namespace: cnpg-system diff --git a/config/rbac/metrics_auth_role_binding.yaml b/config/rbac/metrics_auth_role_binding.yaml index e775d67..a41825d 100644 --- a/config/rbac/metrics_auth_role_binding.yaml +++ b/config/rbac/metrics_auth_role_binding.yaml @@ -8,5 +8,5 @@ roleRef: name: metrics-auth-role subjects: - kind: ServiceAccount - name: controller-manager - namespace: system + name: plugin-barman-cloud + namespace: cnpg-system diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml index ef7dff7..99e9a5e 100644 --- a/config/rbac/role_binding.yaml +++ b/config/rbac/role_binding.yaml @@ -11,5 +11,5 @@ roleRef: name: manager-role subjects: - kind: ServiceAccount - name: controller-manager - namespace: system + name: plugin-barman-cloud + namespace: cnpg-system diff --git a/config/rbac/service_account.yaml b/config/rbac/service_account.yaml index 30bf270..ea0615e 100644 --- a/config/rbac/service_account.yaml +++ b/config/rbac/service_account.yaml @@ -4,5 +4,4 @@ metadata: labels: app.kubernetes.io/name: plugin-barman-cloud app.kubernetes.io/managed-by: kustomize - name: controller-manager - namespace: system + name: plugin-barman-cloud diff --git a/docs/examples/cluster-example.yaml b/docs/examples/cluster-example.yaml new file mode 100644 index 0000000..ea6a103 --- /dev/null +++ b/docs/examples/cluster-example.yaml @@ -0,0 +1,12 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: cluster-example +spec: + instances: 3 + + plugins: + - name: barman-cloud.cloudnative-pg.io + + storage: + size: 1Gi diff --git a/go.mod b/go.mod index 26ababc..12cb241 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,11 @@ require ( github.com/cloudnative-pg/machinery v0.0.0-20241001075747-34c8797af80f github.com/onsi/ginkgo/v2 v2.20.2 github.com/onsi/gomega v1.34.2 + github.com/sourcegraph/conc v0.3.0 + github.com/spf13/cobra v1.8.1 + github.com/spf13/viper v1.19.0 google.golang.org/grpc v1.67.1 + k8s.io/api v0.31.1 k8s.io/apimachinery v0.31.1 k8s.io/client-go v0.31.1 sigs.k8s.io/controller-runtime v0.19.0 @@ -75,12 +79,10 @@ require ( github.com/robfig/cron v1.2.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect - github.com/sourcegraph/conc v0.3.0 // indirect + github.com/snorwin/jsonpatch v1.5.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect - github.com/spf13/cobra v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/spf13/viper v1.19.0 // indirect github.com/stoewer/go-strcase v1.3.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/thoas/go-funk v0.9.3 // indirect @@ -112,7 +114,6 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/api v0.31.1 // indirect k8s.io/apiextensions-apiserver v0.31.0 // indirect k8s.io/apiserver v0.31.0 // indirect k8s.io/component-base v0.31.0 // indirect diff --git a/go.sum b/go.sum index d519cac..e182c0a 100644 --- a/go.sum +++ b/go.sum @@ -43,6 +43,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-faker/faker/v4 v4.4.1 h1:LY1jDgjVkBZWIhATCt+gkl0x9i/7wC61gZx73GTFb+Q= +github.com/go-faker/faker/v4 v4.4.1/go.mod h1:HRLrjis+tYsbFtIHufEPTAIzcZiRu0rS9EYl2Ccwme4= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -155,6 +157,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/snorwin/jsonpatch v1.5.0 h1:0m56YSt9cHiJOn8U+OcqdPGcDQZmhPM/zsG7Dv5QQP0= +github.com/snorwin/jsonpatch v1.5.0/go.mod h1:e0IDKlyFBLTFPqM0wa79dnMwjMs3XFvmKcrgCRpDqok= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= diff --git a/internal/cnpgi/instance/identity.go b/internal/cnpgi/instance/identity.go index 69ede5e..1e29d6b 100644 --- a/internal/cnpgi/instance/identity.go +++ b/internal/cnpgi/instance/identity.go @@ -8,6 +8,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" barmancloudv1 "github.com/cloudnative-pg/plugin-barman-cloud/api/v1" + "github.com/cloudnative-pg/plugin-barman-cloud/internal/cnpgi/metadata" ) // IdentityImplementation implements IdentityServer @@ -22,7 +23,7 @@ func (i IdentityImplementation) GetPluginMetadata( _ context.Context, _ *identity.GetPluginMetadataRequest, ) (*identity.GetPluginMetadataResponse, error) { - return &Data, nil + return &metadata.Data, nil } // GetPluginCapabilities implements IdentityServer diff --git a/internal/cnpgi/instance/constants.go b/internal/cnpgi/metadata/constants.go similarity index 97% rename from internal/cnpgi/instance/constants.go rename to internal/cnpgi/metadata/constants.go index 811a698..02361e2 100644 --- a/internal/cnpgi/instance/constants.go +++ b/internal/cnpgi/metadata/constants.go @@ -1,4 +1,4 @@ -package instance +package metadata import "github.com/cloudnative-pg/cnpg-i/pkg/identity" diff --git a/internal/cnpgi/metadata/doc.go b/internal/cnpgi/metadata/doc.go new file mode 100644 index 0000000..e44e679 --- /dev/null +++ b/internal/cnpgi/metadata/doc.go @@ -0,0 +1,3 @@ +// Package metadata contains the common metadata on the operator +// and on the instance manager +package metadata diff --git a/internal/cnpgi/operator/constants.go b/internal/cnpgi/operator/constants.go deleted file mode 100644 index 08f1e0f..0000000 --- a/internal/cnpgi/operator/constants.go +++ /dev/null @@ -1,18 +0,0 @@ -package operator - -import "github.com/cloudnative-pg/cnpg-i/pkg/identity" - -// PluginName is the name of this plugin -const PluginName = "operator.barman-cloud.cloudnative-pg.io" - -// Data is the metadata of this plugin. -var Data = identity.GetPluginMetadataResponse{ - Name: PluginName, - Version: "0.0.1", - DisplayName: "BarmanCloudOperator", - ProjectUrl: "https://github.com/cloudnative-pg/plugin-barman-cloud", - RepositoryUrl: "https://github.com/cloudnative-pg/plugin-barman-cloud", - License: "APACHE 2.0", - LicenseUrl: "https://github.com/cloudnative-pg/plugin-barman-cloud/LICENSE", - Maturity: "alpha", -} diff --git a/internal/cnpgi/operator/identity.go b/internal/cnpgi/operator/identity.go index 040a2ee..fd933ce 100644 --- a/internal/cnpgi/operator/identity.go +++ b/internal/cnpgi/operator/identity.go @@ -4,6 +4,8 @@ import ( "context" "github.com/cloudnative-pg/cnpg-i/pkg/identity" + + "github.com/cloudnative-pg/plugin-barman-cloud/internal/cnpgi/metadata" ) // IdentityImplementation is the implementation of the CNPG-i @@ -17,7 +19,7 @@ func (i IdentityImplementation) GetPluginMetadata( _ context.Context, _ *identity.GetPluginMetadataRequest, ) (*identity.GetPluginMetadataResponse, error) { - return &Data, nil + return &metadata.Data, nil } // GetPluginCapabilities implements identity diff --git a/internal/cnpgi/operator/lifecycle.go b/internal/cnpgi/operator/lifecycle.go new file mode 100644 index 0000000..35e99e0 --- /dev/null +++ b/internal/cnpgi/operator/lifecycle.go @@ -0,0 +1,79 @@ +package operator + +import ( + "context" + "errors" + + "github.com/cloudnative-pg/cnpg-i-machinery/pkg/pluginhelper/decoder" + "github.com/cloudnative-pg/cnpg-i-machinery/pkg/pluginhelper/object" + "github.com/cloudnative-pg/cnpg-i/pkg/lifecycle" + "github.com/cloudnative-pg/machinery/pkg/log" + "github.com/spf13/viper" + corev1 "k8s.io/api/core/v1" +) + +// LifecycleImplementation is the implementation of the lifecycle handler +type LifecycleImplementation struct { + lifecycle.UnimplementedOperatorLifecycleServer +} + +// GetCapabilities exposes the lifecycle capabilities +func (impl LifecycleImplementation) GetCapabilities( + _ context.Context, + _ *lifecycle.OperatorLifecycleCapabilitiesRequest, +) (*lifecycle.OperatorLifecycleCapabilitiesResponse, error) { + return &lifecycle.OperatorLifecycleCapabilitiesResponse{ + LifecycleCapabilities: []*lifecycle.OperatorLifecycleCapabilities{ + { + Group: "", + Kind: "Pod", + OperationTypes: []*lifecycle.OperatorOperationType{ + { + Type: lifecycle.OperatorOperationType_TYPE_CREATE, + }, + { + Type: lifecycle.OperatorOperationType_TYPE_PATCH, + }, + }, + }, + }, + }, nil +} + +// LifecycleHook is called when creating Kubernetes services +func (impl LifecycleImplementation) LifecycleHook( + ctx context.Context, + request *lifecycle.OperatorLifecycleRequest, +) (*lifecycle.OperatorLifecycleResponse, error) { + contextLogger := log.FromContext(ctx).WithName("plugin-barman-cloud-lifecycle") + + operation := request.GetOperationType().GetType().Enum() + if operation == nil { + return nil, errors.New("no operation set") + } + + pod, err := decoder.DecodePodJSON(request.GetObjectDefinition()) + if err != nil { + return nil, err + } + + mutatedPod := pod.DeepCopy() + err = object.InjectPluginSidecar(mutatedPod, &corev1.Container{ + Name: "plugin-barman-cloud", + Image: viper.GetString("sidecar-image"), + }, false) + if err != nil { + return nil, err + } + + patch, err := object.CreatePatch(mutatedPod, pod) + if err != nil { + return nil, err + } + + // TODO: change to debug + contextLogger.Info("generated patch", "content", string(patch)) + return &lifecycle.OperatorLifecycleResponse{ + JsonPatch: patch, + }, nil +} diff --git a/internal/cnpgi/operator/start.go b/internal/cnpgi/operator/start.go index af745f9..21cd29c 100644 --- a/internal/cnpgi/operator/start.go +++ b/internal/cnpgi/operator/start.go @@ -1,26 +1,21 @@ package operator import ( - "context" - "github.com/cloudnative-pg/cnpg-i-machinery/pkg/pluginhelper/http" - "github.com/cloudnative-pg/cnpg-i/pkg/identity" + "github.com/cloudnative-pg/cnpg-i/pkg/lifecycle" "github.com/cloudnative-pg/cnpg-i/pkg/reconciler" + "github.com/spf13/cobra" "google.golang.org/grpc" ) -// CNPGI is the implementation of the Operator plugin -type CNPGI struct{} - -// Start starts the GRPC server -func (c *CNPGI) Start(ctx context.Context) error { +// NewCommand creates the command to start the GRPC server +// of the operator plugin +func NewCommand() *cobra.Command { cmd := http.CreateMainCmd(IdentityImplementation{}, func(server *grpc.Server) error { - // Register the declared implementations - identity.RegisterIdentityServer(server, IdentityImplementation{}) reconciler.RegisterReconcilerHooksServer(server, ReconcilerImplementation{}) + lifecycle.RegisterOperatorLifecycleServer(server, LifecycleImplementation{}) return nil }) - cmd.Use = "plugin-operator" - - return cmd.ExecuteContext(ctx) //nolint:wrapcheck + cmd.Use = "plugin" + return cmd } diff --git a/internal/manager/manager.go b/internal/manager/manager.go new file mode 100644 index 0000000..b339f94 --- /dev/null +++ b/internal/manager/manager.go @@ -0,0 +1,161 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package manager contains the implementation of the ObjectStore controller manager +package manager + +import ( + "context" + "crypto/tls" + "flag" + + // +kubebuilder:scaffold:imports + "github.com/spf13/viper" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + barmancloudv1 "github.com/cloudnative-pg/plugin-barman-cloud/api/v1" + "github.com/cloudnative-pg/plugin-barman-cloud/internal/controller" + + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + // to ensure that exec-entrypoint and run can make use of them. + _ "k8s.io/client-go/plugin/pkg/client/auth" +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + + utilruntime.Must(barmancloudv1.AddToScheme(scheme)) + // +kubebuilder:scaffold:scheme +} + +// Start starts the manager +func Start(ctx context.Context) error { + var tlsOpts []func(*tls.Config) + + opts := zap.Options{ + Development: true, + } + opts.BindFlags(flag.CommandLine) + flag.Parse() + + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + + // if the enable-http2 flag is false (the default), http/2 should be disabled + // due to its vulnerabilities. More specifically, disabling http/2 will + // prevent from being vulnerable to the HTTP/2 Stream Cancellation and + // Rapid Reset CVEs. For more information see: + // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 + // - https://github.com/advisories/GHSA-4374-p667-p6c8 + disableHTTP2 := func(c *tls.Config) { + setupLog.Info("disabling http/2") + c.NextProtos = []string{"http/1.1"} + } + + if !viper.GetBool("enable-http2") { + tlsOpts = append(tlsOpts, disableHTTP2) + } + + webhookServer := webhook.NewServer(webhook.Options{ + TLSOpts: tlsOpts, + }) + + // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. + // More info: + // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.0/pkg/metrics/server + // - https://book.kubebuilder.io/reference/metrics.html + metricsServerOptions := metricsserver.Options{ + BindAddress: viper.GetString("metrics-bind-address"), + SecureServing: viper.GetBool("metrics-secure"), + // TODO(user): TLSOpts is used to allow configuring the TLS config used for the server. If certificates are + // not provided, self-signed certificates will be generated by default. This option is not recommended for + // production environments as self-signed certificates do not offer the same level of trust and security + // as certificates issued by a trusted Certificate Authority (CA). The primary risk is potentially allowing + // unauthorized access to sensitive metrics data. Consider replacing with CertDir, CertName, and KeyName + // to provide certificates, ensuring the server communicates using trusted and secure certificates. + TLSOpts: tlsOpts, + } + + if viper.GetBool("metrics-secure") { + // FilterProvider is used to protect the metrics endpoint with authn/authz. + // These configurations ensure that only authorized users and service accounts + // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: + // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.0/pkg/metrics/filters#WithAuthenticationAndAuthorization + metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization + } + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + Metrics: metricsServerOptions, + WebhookServer: webhookServer, + HealthProbeBindAddress: viper.GetString("health-probe-bind-address"), + LeaderElection: viper.GetBool("leader-elect"), + LeaderElectionID: "822e3f5c.cnpg.io", + // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily + // when the Manager ends. This requires the binary to immediately end when the + // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly + // speeds up voluntary leader transitions as the new leader don't have to wait + // LeaseDuration time first. + // + // In the default scaffold provided, the program ends immediately after + // the manager stops, so would be fine to enable this option. However, + // if you are doing or is intended to do any operation such as perform cleanups + // after the manager stops then its usage might be unsafe. + // LeaderElectionReleaseOnCancel: true, + }) + if err != nil { + setupLog.Error(err, "unable to start manager") + return err + } + + if err = (&controller.ObjectStoreReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "ObjectStore") + return err + } + // +kubebuilder:scaffold:builder + + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up health check") + return err + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up ready check") + return err + } + + setupLog.Info("starting manager") + if err := mgr.Start(ctx); err != nil { + setupLog.Error(err, "problem running manager") + return err + } + + return nil +} diff --git a/kubernetes/certificate-issuer.yaml b/kubernetes/certificate-issuer.yaml new file mode 100644 index 0000000..8d3d6ee --- /dev/null +++ b/kubernetes/certificate-issuer.yaml @@ -0,0 +1,6 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: selfsigned-issuer +spec: + selfSigned: {} diff --git a/kubernetes/client-certificate.yaml b/kubernetes/client-certificate.yaml new file mode 100644 index 0000000..a2893b2 --- /dev/null +++ b/kubernetes/client-certificate.yaml @@ -0,0 +1,19 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: barman-cloud-client +spec: + secretName: barman-cloud-client-tls + + commonName: "barman-cloud-client" + duration: 2160h # 90d + renewBefore: 360h # 15d + + isCA: false + usages: + - client auth + + issuerRef: + name: selfsigned-issuer + kind: Issuer + group: cert-manager.io diff --git a/kubernetes/deployment.yaml b/kubernetes/deployment.yaml new file mode 100644 index 0000000..29820bb --- /dev/null +++ b/kubernetes/deployment.yaml @@ -0,0 +1,50 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: barman-cloud + name: barman-cloud +spec: + replicas: 1 + selector: + matchLabels: + app: barman-cloud + strategy: {} + template: + metadata: + labels: + app: barman-cloud + spec: + serviceAccountName: plugin-barman-cloud + containers: + - image: plugin-barman-cloud:latest + name: barman-cloud + ports: + - containerPort: 9090 + protocol: TCP + env: + - name: SIDECAR_IMAGE + valueFrom: + secretKeyRef: + key: SIDECAR_IMAGE + name: plugin-barman-cloud + args: + - operator + - --server-cert=/server/tls.crt + - --server-key=/server/tls.key + - --client-cert=/client/tls.crt + - --server-address=:9090 + - --leader-elect + volumeMounts: + - mountPath: /server + name: server + - mountPath: /client + name: client + resources: {} + volumes: + - name: server + secret: + secretName: barman-cloud-server-tls + - name: client + secret: + secretName: barman-cloud-client-tls diff --git a/kubernetes/kustomization.yaml b/kubernetes/kustomization.yaml new file mode 100644 index 0000000..eebeee0 --- /dev/null +++ b/kubernetes/kustomization.yaml @@ -0,0 +1,17 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: cnpg-system +resources: +- certificate-issuer.yaml +- client-certificate.yaml +- deployment.yaml +- server-certificate.yaml +- service.yaml +- ../config/crd +- ../config/rbac +images: +- name: plugin-barman-cloud +secretGenerator: +- literals: + - SIDECAR_IMAGE=plugin-sidecar + name: plugin-barman-cloud diff --git a/kubernetes/server-certificate.yaml b/kubernetes/server-certificate.yaml new file mode 100644 index 0000000..393c5f1 --- /dev/null +++ b/kubernetes/server-certificate.yaml @@ -0,0 +1,21 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: barman-cloud-server +spec: + secretName: barman-cloud-server-tls + commonName: barman-cloud + dnsNames: + - barman-cloud + + duration: 2160h # 90d + renewBefore: 360h # 15d + + isCA: false + usages: + - server auth + + issuerRef: + name: selfsigned-issuer + kind: Issuer + group: cert-manager.io diff --git a/kubernetes/service.yaml b/kubernetes/service.yaml new file mode 100644 index 0000000..3987b00 --- /dev/null +++ b/kubernetes/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: barman-cloud + cnpg.io/pluginName: barman-cloud.cloudnative-pg.io + annotations: + cnpg.io/pluginClientSecret: barman-cloud-client-tls + cnpg.io/pluginServerSecret: barman-cloud-server-tls + cnpg.io/pluginPort: "9090" + name: barman-cloud +spec: + ports: + - port: 9090 + protocol: TCP + targetPort: 9090 + selector: + app: barman-cloud diff --git a/scripts/run.sh b/scripts/run.sh new file mode 100755 index 0000000..994c6e9 --- /dev/null +++ b/scripts/run.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -eu + +cd "$(dirname "$0")/.." || exit + +if [ -f .env ]; then + source .env +fi + +current_context=$(kubectl config view --raw -o json | jq -r '."current-context"' | sed "s/kind-//") +operator_image=$(KIND_CLUSTER_NAME="$current_context" KO_DOCKER_REPO=kind.local ko build -BP ./cmd/operator) +instance_image=$(KIND_CLUSTER_NAME="$current_context" KO_DOCKER_REPO=kind.local ko build -BP ./cmd/instance) + +( + cd kubernetes; + kustomize edit set image "plugin-barman-cloud=$operator_image" + kustomize edit set secret plugin-barman-cloud "--from-literal=SIDECAR_IMAGE=$instance_image" +) + +# Now we deploy the plugin inside the `cnpg-system` workspace +kubectl apply -k kubernetes/