From ea6ee30d2ea30f9e9df22002ce5f5a68fcb37ade Mon Sep 17 00:00:00 2001 From: Leonardo Cecchi Date: Wed, 2 Oct 2024 13:21:04 +0200 Subject: [PATCH 1/4] feat: sidecar injection and loading (#22) Signed-off-by: Leonardo Cecchi --- .golangci.yml | 2 +- cmd/instance/main.go | 129 ++++++------------ cmd/operator/main.go | 4 +- docs/examples/cluster-example.yaml | 2 + go.mod | 12 +- go.sum | 24 ++-- internal/cnpgi/instance/manager.go | 88 ++++++++++++ internal/cnpgi/instance/start.go | 13 +- internal/cnpgi/metadata/constants.go | 2 +- internal/cnpgi/operator/lifecycle.go | 32 ++++- internal/{ => operator}/controller/doc.go | 0 .../controller/objectstore_controller.go | 0 .../controller/objectstore_controller_test.go | 0 .../{ => operator}/controller/suite_test.go | 4 +- internal/{ => operator}/manager/manager.go | 10 +- kubernetes/kustomization.yaml | 4 +- 16 files changed, 192 insertions(+), 134 deletions(-) create mode 100644 internal/cnpgi/instance/manager.go rename internal/{ => operator}/controller/doc.go (100%) rename internal/{ => operator}/controller/objectstore_controller.go (100%) rename internal/{ => operator}/controller/objectstore_controller_test.go (100%) rename internal/{ => operator}/controller/suite_test.go (95%) rename internal/{ => operator}/manager/manager.go (96%) diff --git a/.golangci.yml b/.golangci.yml index 67f4b07..5148a0b 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -131,4 +131,4 @@ issues: - test exclude-files: - zz_generated.* - - internal/controller/suite_test.go + - internal/operator/controller/suite_test.go diff --git a/cmd/instance/main.go b/cmd/instance/main.go index 4eed69c..a228ce1 100644 --- a/cmd/instance/main.go +++ b/cmd/instance/main.go @@ -1,107 +1,58 @@ -// Package main is the implementation of the CNPG-i PostgreSQL sidecar +// Package main is the entrypoint of operator plugin package main import ( - "errors" + "fmt" "os" - cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/runtime" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/cloudnative-pg/machinery/pkg/log" + "github.com/spf13/cobra" + "github.com/spf13/viper" - barmancloudv1 "github.com/cloudnative-pg/plugin-barman-cloud/api/v1" "github.com/cloudnative-pg/plugin-barman-cloud/internal/cnpgi/instance" ) -var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") -) - -func init() { - utilruntime.Must(clientgoscheme.AddToScheme(scheme)) - - utilruntime.Must(barmancloudv1.AddToScheme(scheme)) - // +kubebuilder:scaffold:scheme -} - func main() { - setupLog.Info("Starting barman cloud instance plugin") - namespace := mustGetEnv("NAMESPACE") - boName := mustGetEnv("BARMAN_OBJECT_NAME") - clusterName := mustGetEnv("CLUSTER_NAME") - instanceName := mustGetEnv("INSTANCE_NAME") + cobra.EnableTraverseRunHooks = true - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - Cache: cache.Options{ - ByObject: map[client.Object]cache.ByObject{ - &barmancloudv1.ObjectStore{}: { - Field: fields.OneTermEqualSelector("metadata.name", boName), - Namespaces: map[string]cache.Config{ - namespace: {}, - }, - }, - &cnpgv1.Cluster{}: { - Field: fields.OneTermEqualSelector("metadata.name", clusterName), - Namespaces: map[string]cache.Config{ - namespace: {}, - }, - }, - }, + logFlags := &log.Flags{} + rootCmd := &cobra.Command{ + Use: "instance", + Short: "Starts the Barman Cloud CNPG-i sidecar plugin", + PersistentPreRunE: func(_ *cobra.Command, _ []string) error { + logFlags.ConfigureLogging() + return nil + }, + RunE: func(cmd *cobra.Command, _ []string) error { + requiredSettings := []string{ + "namespace", + "barman-object-name", + "cluster-name", + "pod-name", + "spool-directory", + } + + for _, k := range requiredSettings { + if len(viper.GetString(k)) == 0 { + return fmt.Errorf("missing required %s setting", k) + } + } + + return instance.Start(cmd.Context()) }, - }) - if err != nil { - setupLog.Error(err, "unable to start manager") - os.Exit(1) } - if err := mgr.Add(&instance.CNPGI{ - Client: mgr.GetClient(), - ClusterObjectKey: client.ObjectKey{ - Namespace: namespace, - Name: clusterName, - }, - BarmanObjectKey: client.ObjectKey{ - Namespace: namespace, - Name: boName, - }, - InstanceName: instanceName, - // TODO: improve - PGDataPath: mustGetEnv("PGDATA"), - PGWALPath: mustGetEnv("PGWAL"), - SpoolDirectory: mustGetEnv("SPOOL_DIRECTORY"), - ServerCertPath: mustGetEnv("SERVER_CERT"), - ServerKeyPath: mustGetEnv("SERVER_KEY"), - ClientCertPath: mustGetEnv("CLIENT_CERT"), - ServerAddress: mustGetEnv("SERVER_ADDRESS"), - PluginPath: mustGetEnv("PLUGIN_PATH"), - }); err != nil { - setupLog.Error(err, "unable to create CNPGI runnable") - os.Exit(1) - } + logFlags.AddFlags(rootCmd.PersistentFlags()) - if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { - setupLog.Error(err, "problem running manager") + _ = viper.BindEnv("namespace", "NAMESPACE") + _ = viper.BindEnv("barman-object-name", "BARMAN_OBJECT_NAME") + _ = viper.BindEnv("cluster-name", "CLUSTER_NAME") + _ = viper.BindEnv("pod-name", "POD_NAME") + _ = viper.BindEnv("pgdata", "PGDATA") + _ = viper.BindEnv("spool-directory", "SPOOL_DIRECTORY") + + if err := rootCmd.Execute(); err != nil { + fmt.Println(err) os.Exit(1) } } - -func mustGetEnv(envName string) string { - value := os.Getenv(envName) - if value == "" { - setupLog.Error( - errors.New("missing required env variable"), - "while fetching env variables", - "name", - envName, - ) - os.Exit(1) - } - return value -} diff --git a/cmd/operator/main.go b/cmd/operator/main.go index f0954e8..44b5ad9 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -10,10 +10,9 @@ import ( "github.com/sourcegraph/conc/pool" "github.com/spf13/cobra" "github.com/spf13/viper" - ctrl "sigs.k8s.io/controller-runtime" "github.com/cloudnative-pg/plugin-barman-cloud/internal/cnpgi/operator" - "github.com/cloudnative-pg/plugin-barman-cloud/internal/manager" + "github.com/cloudnative-pg/plugin-barman-cloud/internal/operator/manager" ) func main() { @@ -44,7 +43,6 @@ func newOperatorCommand() *cobra.Command { grpcServer := cmd.RunE cmd.RunE = func(cmd *cobra.Command, args []string) error { - ctrl.SetupSignalHandler() operatorPool := pool. New(). WithContext(cmd.Context()). diff --git a/docs/examples/cluster-example.yaml b/docs/examples/cluster-example.yaml index ea6a103..33983b8 100644 --- a/docs/examples/cluster-example.yaml +++ b/docs/examples/cluster-example.yaml @@ -7,6 +7,8 @@ spec: plugins: - name: barman-cloud.cloudnative-pg.io + parameters: + barmanObjectStore: minio-store storage: size: 1Gi diff --git a/go.mod b/go.mod index 12cb241..732001d 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,9 @@ go 1.22.0 require ( github.com/cloudnative-pg/barman-cloud v0.0.0-20240924124724-92831d48562a - github.com/cloudnative-pg/cloudnative-pg v1.24.0 + github.com/cloudnative-pg/cloudnative-pg v1.24.1-0.20241001084914-829808376542 github.com/cloudnative-pg/cnpg-i v0.0.0-20240924030516-c5636170f248 - github.com/cloudnative-pg/cnpg-i-machinery v0.0.0-20240926153929-09e2c6f6689b + github.com/cloudnative-pg/cnpg-i-machinery v0.0.0-20241002070940-e5495e9c5ed6 github.com/cloudnative-pg/machinery v0.0.0-20241001075747-34c8797af80f github.com/onsi/ginkgo/v2 v2.20.2 github.com/onsi/gomega v1.34.2 @@ -114,12 +114,12 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.31.0 // indirect - k8s.io/apiserver v0.31.0 // indirect - k8s.io/component-base v0.31.0 // indirect + k8s.io/apiextensions-apiserver v0.31.1 // indirect + k8s.io/apiserver v0.31.1 // indirect + k8s.io/component-base v0.31.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 // indirect - k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 // indirect + k8s.io/utils v0.0.0-20240921022957-49e7df575cb6 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect diff --git a/go.sum b/go.sum index e182c0a..ade4eed 100644 --- a/go.sum +++ b/go.sum @@ -16,12 +16,12 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cloudnative-pg/barman-cloud v0.0.0-20240924124724-92831d48562a h1:0v1ML9Eibfq3helbT9GtU0EstqFtG91k/MPO9azY5ME= github.com/cloudnative-pg/barman-cloud v0.0.0-20240924124724-92831d48562a/go.mod h1:Jm0tOp5oB7utpt8wz6RfSv31h1mThOtffjfyxVupriE= -github.com/cloudnative-pg/cloudnative-pg v1.24.0 h1:lY9IP/Gnh5ogNcFGoPnbD2eOiHdSdbdEp0PaUdOAMDQ= -github.com/cloudnative-pg/cloudnative-pg v1.24.0/go.mod h1:n7Qqax6os+x3+7Qu/GojUUeKlL1ELGV63dcO/tzIfB4= +github.com/cloudnative-pg/cloudnative-pg v1.24.1-0.20241001084914-829808376542 h1:IXf5lj+m4CBqzckQ9L/9hJ01JUoVw5N0FuPex0sVdVo= +github.com/cloudnative-pg/cloudnative-pg v1.24.1-0.20241001084914-829808376542/go.mod h1:L8M+kTGpz/eaLZj46+4sARvO/vDYlo/m1xOigI/ghBA= github.com/cloudnative-pg/cnpg-i v0.0.0-20240924030516-c5636170f248 h1:eUGzb7YNjVLilwhgZoe4hDOO70fci3oqb/ZzQFbN3xg= github.com/cloudnative-pg/cnpg-i v0.0.0-20240924030516-c5636170f248/go.mod h1:K9/4eAT3rh2bKIWyujoN8BIPRXa4d1Ls+eBY8PE8y6w= -github.com/cloudnative-pg/cnpg-i-machinery v0.0.0-20240926153929-09e2c6f6689b h1:T9G61tzOBoB5yvlDPULUoiUl6QxPmti3pkNFhQYGGQY= -github.com/cloudnative-pg/cnpg-i-machinery v0.0.0-20240926153929-09e2c6f6689b/go.mod h1:dV1+nE7jWENm/fcnKBeKsaScMz685rQPbPCCDydJgsY= +github.com/cloudnative-pg/cnpg-i-machinery v0.0.0-20241002070940-e5495e9c5ed6 h1:C4CU5fBTYTiJBPDqcgHpXSc5IvRTy+5KTaFZzdKHfAQ= +github.com/cloudnative-pg/cnpg-i-machinery v0.0.0-20241002070940-e5495e9c5ed6/go.mod h1:mHEVy/Guae+rij1qlgwHg+lyFKDX48qjTL4lAqE7OJs= github.com/cloudnative-pg/machinery v0.0.0-20241001075747-34c8797af80f h1:RgPmQJkuSu3eTdfd4T2K95RYQi57LHB2+Jfsu/faKOM= github.com/cloudnative-pg/machinery v0.0.0-20241001075747-34c8797af80f/go.mod h1:bWp1Es5zlxElg4Z/c5f0RKOkDcyNvDHdYIvNcPQU4WM= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -286,22 +286,22 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/api v0.31.1 h1:Xe1hX/fPW3PXYYv8BlozYqw63ytA92snr96zMW9gWTU= k8s.io/api v0.31.1/go.mod h1:sbN1g6eY6XVLeqNsZGLnI5FwVseTrZX7Fv3O26rhAaI= -k8s.io/apiextensions-apiserver v0.31.0 h1:fZgCVhGwsclj3qCw1buVXCV6khjRzKC5eCFt24kyLSk= -k8s.io/apiextensions-apiserver v0.31.0/go.mod h1:b9aMDEYaEe5sdK+1T0KU78ApR/5ZVp4i56VacZYEHxk= +k8s.io/apiextensions-apiserver v0.31.1 h1:L+hwULvXx+nvTYX/MKM3kKMZyei+UiSXQWciX/N6E40= +k8s.io/apiextensions-apiserver v0.31.1/go.mod h1:tWMPR3sgW+jsl2xm9v7lAyRF1rYEK71i9G5dRtkknoQ= k8s.io/apimachinery v0.31.1 h1:mhcUBbj7KUjaVhyXILglcVjuS4nYXiwC+KKFBgIVy7U= k8s.io/apimachinery v0.31.1/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= -k8s.io/apiserver v0.31.0 h1:p+2dgJjy+bk+B1Csz+mc2wl5gHwvNkC9QJV+w55LVrY= -k8s.io/apiserver v0.31.0/go.mod h1:KI9ox5Yu902iBnnyMmy7ajonhKnkeZYJhTZ/YI+WEMk= +k8s.io/apiserver v0.31.1 h1:Sars5ejQDCRBY5f7R3QFHdqN3s61nhkpaX8/k1iEw1c= +k8s.io/apiserver v0.31.1/go.mod h1:lzDhpeToamVZJmmFlaLwdYZwd7zB+WYRYIboqA1kGxM= k8s.io/client-go v0.31.1 h1:f0ugtWSbWpxHR7sjVpQwuvw9a3ZKLXX0u0itkFXufb0= k8s.io/client-go v0.31.1/go.mod h1:sKI8871MJN2OyeqRlmA4W4KM9KBdBUpDLu/43eGemCg= -k8s.io/component-base v0.31.0 h1:/KIzGM5EvPNQcYgwq5NwoQBaOlVFrghoVGr8lG6vNRs= -k8s.io/component-base v0.31.0/go.mod h1:TYVuzI1QmN4L5ItVdMSXKvH7/DtvIuas5/mm8YT3rTo= +k8s.io/component-base v0.31.1 h1:UpOepcrX3rQ3ab5NB6g5iP0tvsgJWzxTyAo20sgYSy8= +k8s.io/component-base v0.31.1/go.mod h1:WGeaw7t/kTsqpVTaCoVEtillbqAhF2/JgvO0LDOMa0w= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 h1:1dWzkmJrrprYvjGwh9kEUxmcUV/CtNU8QM7h1FLWQOo= k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38/go.mod h1:coRQXBK9NxO98XUv3ZD6AK3xzHCxV6+b7lrquKwaKzA= -k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 h1:b2FmK8YH+QEwq/Sy2uAEhmqL5nPfGYbJOcaqjeYYZoA= -k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20240921022957-49e7df575cb6 h1:MDF6h2H/h4tbzmtIKTuctcwZmY0tY9mD9fNT47QO6HI= +k8s.io/utils v0.0.0-20240921022957-49e7df575cb6/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 h1:2770sDpzrjjsAtVhSeUFseziht227YAWYHLGNM8QPwY= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/controller-runtime v0.19.0 h1:nWVM7aq+Il2ABxwiCizrVDSlmDcshi9llbaFbC0ji/Q= diff --git a/internal/cnpgi/instance/manager.go b/internal/cnpgi/instance/manager.go new file mode 100644 index 0000000..18e8ff4 --- /dev/null +++ b/internal/cnpgi/instance/manager.go @@ -0,0 +1,88 @@ +package instance + +import ( + "context" + "os" + "path" + + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + "github.com/spf13/viper" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + barmancloudv1 "github.com/cloudnative-pg/plugin-barman-cloud/api/v1" +) + +var scheme = runtime.NewScheme() + +func init() { + utilruntime.Must(barmancloudv1.AddToScheme(scheme)) + utilruntime.Must(cnpgv1.AddToScheme(scheme)) +} + +// Start starts the sidecar informers and CNPG-i server +func Start(ctx context.Context) error { + setupLog := log.FromContext(ctx) + setupLog.Info("Starting barman cloud instance plugin") + namespace := viper.GetString("namespace") + boName := viper.GetString("barman-object-name") + clusterName := viper.GetString("cluster-name") + podName := viper.GetString("pod-name") + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + Cache: cache.Options{ + ByObject: map[client.Object]cache.ByObject{ + &barmancloudv1.ObjectStore{}: { + Field: fields.OneTermEqualSelector("metadata.name", boName), + Namespaces: map[string]cache.Config{ + namespace: {}, + }, + }, + &cnpgv1.Cluster{}: { + Field: fields.OneTermEqualSelector("metadata.name", clusterName), + Namespaces: map[string]cache.Config{ + namespace: {}, + }, + }, + }, + }, + }) + if err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } + + if err := mgr.Add(&CNPGI{ + Client: mgr.GetClient(), + ClusterObjectKey: client.ObjectKey{ + Namespace: namespace, + Name: clusterName, + }, + BarmanObjectKey: client.ObjectKey{ + Namespace: namespace, + Name: boName, + }, + InstanceName: podName, + // TODO: improve + PGDataPath: viper.GetString("pgdata"), + PGWALPath: path.Join(viper.GetString("pgdata"), "pg_wal"), + SpoolDirectory: viper.GetString("spool-directory"), + PluginPath: viper.GetString("plugin-path"), + }); err != nil { + setupLog.Error(err, "unable to create CNPGI runnable") + return err + } + + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + setupLog.Error(err, "problem running manager") + return err + } + + return nil +} diff --git a/internal/cnpgi/instance/start.go b/internal/cnpgi/instance/start.go index f2f549f..3df9093 100644 --- a/internal/cnpgi/instance/start.go +++ b/internal/cnpgi/instance/start.go @@ -18,11 +18,6 @@ type CNPGI struct { PGDataPath string PGWALPath string SpoolDirectory string - ServerCertPath string - ServerKeyPath string - ClientCertPath string - // mutually exclusive with pluginPath - ServerAddress string // mutually exclusive with serverAddress PluginPath string InstanceName string @@ -49,12 +44,8 @@ func (c *CNPGI) Start(ctx context.Context) error { Client: c.Client, BarmanObjectKey: c.BarmanObjectKey, }, - Enrichers: []http.ServerEnricher{enrich}, - ServerCertPath: c.ServerCertPath, - ServerKeyPath: c.ServerKeyPath, - ClientCertPath: c.ClientCertPath, - ServerAddress: c.ServerAddress, - PluginPath: c.PluginPath, + Enrichers: []http.ServerEnricher{enrich}, + PluginPath: c.PluginPath, } return srv.Start(ctx) diff --git a/internal/cnpgi/metadata/constants.go b/internal/cnpgi/metadata/constants.go index 02361e2..2fa2fbf 100644 --- a/internal/cnpgi/metadata/constants.go +++ b/internal/cnpgi/metadata/constants.go @@ -4,7 +4,7 @@ import "github.com/cloudnative-pg/cnpg-i/pkg/identity" // PluginName is the name of the plugin from the instance manager // Point-of-view -const PluginName = "instance.barman-cloud.cloudnative-pg.io" +const PluginName = "barman-cloud.cloudnative-pg.io" // Data is the metadata of this plugin. var Data = identity.GetPluginMetadataResponse{ diff --git a/internal/cnpgi/operator/lifecycle.go b/internal/cnpgi/operator/lifecycle.go index 35e99e0..8781cdd 100644 --- a/internal/cnpgi/operator/lifecycle.go +++ b/internal/cnpgi/operator/lifecycle.go @@ -4,12 +4,15 @@ import ( "context" "errors" + "github.com/cloudnative-pg/cnpg-i-machinery/pkg/pluginhelper/common" "github.com/cloudnative-pg/cnpg-i-machinery/pkg/pluginhelper/decoder" "github.com/cloudnative-pg/cnpg-i-machinery/pkg/pluginhelper/object" "github.com/cloudnative-pg/cnpg-i/pkg/lifecycle" "github.com/cloudnative-pg/machinery/pkg/log" "github.com/spf13/viper" corev1 "k8s.io/api/core/v1" + + "github.com/cloudnative-pg/plugin-barman-cloud/internal/cnpgi/metadata" ) // LifecycleImplementation is the implementation of the lifecycle handler @@ -52,16 +55,40 @@ func (impl LifecycleImplementation) LifecycleHook( return nil, errors.New("no operation set") } + cluster, err := decoder.DecodeClusterJSON(request.GetClusterDefinition()) + if err != nil { + return nil, err + } + pod, err := decoder.DecodePodJSON(request.GetObjectDefinition()) if err != nil { return nil, err } + helper := common.NewPlugin( + *cluster, + metadata.PluginName, + ) + + // TODO: Validation of the plugin configuration + mutatedPod := pod.DeepCopy() err = object.InjectPluginSidecar(mutatedPod, &corev1.Container{ Name: "plugin-barman-cloud", Image: viper.GetString("sidecar-image"), - }, false) + Env: []corev1.EnvVar{ + { + Name: "BARMAN_OBJECT_NAME", + Value: helper.Parameters["barmanObjectStore"], + }, + { + // TODO: should we really use this one? + // should we mount an emptyDir volume just for that? + Name: "SPOOL_DIRECTORY", + Value: "/controller/wal-restore-spool", + }, + }, + }, true) if err != nil { return nil, err } @@ -71,8 +98,7 @@ func (impl LifecycleImplementation) LifecycleHook( return nil, err } - // TODO: change to debug - contextLogger.Info("generated patch", "content", string(patch)) + contextLogger.Debug("generated patch", "content", string(patch)) return &lifecycle.OperatorLifecycleResponse{ JsonPatch: patch, }, nil diff --git a/internal/controller/doc.go b/internal/operator/controller/doc.go similarity index 100% rename from internal/controller/doc.go rename to internal/operator/controller/doc.go diff --git a/internal/controller/objectstore_controller.go b/internal/operator/controller/objectstore_controller.go similarity index 100% rename from internal/controller/objectstore_controller.go rename to internal/operator/controller/objectstore_controller.go diff --git a/internal/controller/objectstore_controller_test.go b/internal/operator/controller/objectstore_controller_test.go similarity index 100% rename from internal/controller/objectstore_controller_test.go rename to internal/operator/controller/objectstore_controller_test.go diff --git a/internal/controller/suite_test.go b/internal/operator/controller/suite_test.go similarity index 95% rename from internal/controller/suite_test.go rename to internal/operator/controller/suite_test.go index 47b937c..2cbe841 100644 --- a/internal/controller/suite_test.go +++ b/internal/operator/controller/suite_test.go @@ -57,11 +57,11 @@ func TestControllers(t *testing.T) { var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) - ctx, cancel = context.WithCancel(context.TODO()) + ctx, cancel = context.WithCancel(context.TODO()) By("bootstrapping test environment") testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")}, ErrorIfCRDPathMissing: true, // The BinaryAssetsDirectory is only required if you want to run the tests directly diff --git a/internal/manager/manager.go b/internal/operator/manager/manager.go similarity index 96% rename from internal/manager/manager.go rename to internal/operator/manager/manager.go index b339f94..ad37848 100644 --- a/internal/manager/manager.go +++ b/internal/operator/manager/manager.go @@ -23,6 +23,7 @@ import ( "flag" // +kubebuilder:scaffold:imports + "github.com/cloudnative-pg/machinery/pkg/log" "github.com/spf13/viper" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -35,17 +36,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook" barmancloudv1 "github.com/cloudnative-pg/plugin-barman-cloud/api/v1" - "github.com/cloudnative-pg/plugin-barman-cloud/internal/controller" + "github.com/cloudnative-pg/plugin-barman-cloud/internal/operator/controller" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. _ "k8s.io/client-go/plugin/pkg/client/auth" ) -var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") -) +var scheme = runtime.NewScheme() func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) @@ -56,6 +54,8 @@ func init() { // Start starts the manager func Start(ctx context.Context) error { + setupLog := log.FromContext(ctx) + var tlsOpts []func(*tls.Config) opts := zap.Options{ diff --git a/kubernetes/kustomization.yaml b/kubernetes/kustomization.yaml index eebeee0..31d8603 100644 --- a/kubernetes/kustomization.yaml +++ b/kubernetes/kustomization.yaml @@ -11,7 +11,9 @@ resources: - ../config/rbac images: - name: plugin-barman-cloud + newName: kind.local/github.com/cloudnative-pg/plugin-barman-cloud/cmd/operator + newTag: 634ab82d4f8b68ffada6033b7dff817bbed61a2fec8e05f9ebf74f5bedafb0dd secretGenerator: - literals: - - SIDECAR_IMAGE=plugin-sidecar + - SIDECAR_IMAGE=kind.local/github.com/cloudnative-pg/plugin-barman-cloud/cmd/instance:7173a2dcf3ce74e982ca3d35053de40fcec67b31d607a0d12547b4f9d09c535c name: plugin-barman-cloud From 2f62d539c949f344cb5534b7ffbb90860663a106 Mon Sep 17 00:00:00 2001 From: Leonardo Cecchi Date: Wed, 2 Oct 2024 15:03:07 +0200 Subject: [PATCH 2/4] feat: sidecar role and rolebinding (#23) Signed-off-by: Leonardo Cecchi --- Makefile | 2 +- api/v1/zz_generated.deepcopy.go | 3 +- cmd/operator/main.go | 122 +++++++------ .../barmancloud.cnpg.io_objectstores.yaml | 6 +- config/rbac/role.yaml | 14 +- config/rbac/role_binding.yaml | 4 +- .../samples/barmancloud_v1_objectstore.yaml | 9 - docs/examples/cluster-example.yaml | 2 +- docs/examples/minio-store.yaml | 23 +++ docs/minio/minio-client.yaml | 20 +++ docs/minio/minio-delete.sh | 1 + docs/minio/minio-deployment.yaml | 42 +++++ docs/minio/minio-pvc.yaml | 11 ++ docs/minio/minio-secret.yaml | 7 + docs/minio/minio-service.yaml | 11 ++ go.mod | 2 +- internal/cnpgi/operator/config/config.go | 71 ++++++++ internal/cnpgi/operator/config/doc.go | 2 + internal/cnpgi/operator/lifecycle.go | 15 +- .../manager => cnpgi/operator}/manager.go | 30 ++-- internal/cnpgi/operator/reconciler.go | 167 +++++++++++++++++- internal/cnpgi/operator/start.go | 40 ++++- internal/{operator => }/controller/doc.go | 0 .../controller/objectstore_controller.go | 2 + .../controller/objectstore_controller_test.go | 0 .../{operator => }/controller/suite_test.go | 2 +- kubernetes/deployment.yaml | 1 - kubernetes/kustomization.yaml | 4 +- 28 files changed, 501 insertions(+), 112 deletions(-) delete mode 100644 config/samples/barmancloud_v1_objectstore.yaml create mode 100644 docs/examples/minio-store.yaml create mode 100644 docs/minio/minio-client.yaml create mode 100644 docs/minio/minio-delete.sh create mode 100644 docs/minio/minio-deployment.yaml create mode 100644 docs/minio/minio-pvc.yaml create mode 100644 docs/minio/minio-secret.yaml create mode 100644 docs/minio/minio-service.yaml create mode 100644 internal/cnpgi/operator/config/config.go create mode 100644 internal/cnpgi/operator/config/doc.go rename internal/{operator/manager => cnpgi/operator}/manager.go (90%) rename internal/{operator => }/controller/doc.go (100%) rename internal/{operator => }/controller/objectstore_controller.go (91%) rename internal/{operator => }/controller/objectstore_controller_test.go (100%) rename internal/{operator => }/controller/suite_test.go (96%) diff --git a/Makefile b/Makefile index 14795f8..9ef3ba8 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ help: ## Display this help. .PHONY: manifests manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. - $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases + $(CONTROLLER_GEN) rbac:roleName=plugin-barman-cloud crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases .PHONY: generate generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 396ce20..6780b95 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -29,7 +29,7 @@ func (in *ObjectStore) DeepCopyInto(out *ObjectStore) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) out.Status = in.Status } @@ -86,6 +86,7 @@ func (in *ObjectStoreList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ObjectStoreSpec) DeepCopyInto(out *ObjectStoreSpec) { *out = *in + in.Configuration.DeepCopyInto(&out.Configuration) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectStoreSpec. diff --git a/cmd/operator/main.go b/cmd/operator/main.go index 44b5ad9..e40128d 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -2,17 +2,14 @@ package main import ( - "context" "fmt" "os" "github.com/cloudnative-pg/machinery/pkg/log" - "github.com/sourcegraph/conc/pool" "github.com/spf13/cobra" "github.com/spf13/viper" "github.com/cloudnative-pg/plugin-barman-cloud/internal/cnpgi/operator" - "github.com/cloudnative-pg/plugin-barman-cloud/internal/operator/manager" ) func main() { @@ -20,7 +17,15 @@ func main() { logFlags := &log.Flags{} rootCmd := &cobra.Command{ - Use: "plugin-barman-cloud", + Use: "plugin-barman-cloud", + Short: "Starts the BarmanObjectStore reconciler and the Barman Cloud CNPG-i plugin", + RunE: func(cmd *cobra.Command, _ []string) error { + if len(viper.GetString("sidecar-image")) == 0 { + return fmt.Errorf("missing required SIDECAR_IMAGE environment variable") + } + + return operator.Start(cmd.Context()) + }, PersistentPreRunE: func(_ *cobra.Command, _ []string) error { logFlags.ConfigureLogging() return nil @@ -28,61 +33,66 @@ func main() { } logFlags.AddFlags(rootCmd.PersistentFlags()) - rootCmd.AddCommand(newOperatorCommand()) + + rootCmd.Flags().String("metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ + "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") + _ = viper.BindPFlag("metrics-bind-address", rootCmd.Flags().Lookup("metrics-bind-address")) + + rootCmd.Flags().String("health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + _ = viper.BindPFlag("health-probe-bind-address", rootCmd.Flags().Lookup("health-probe-bind-address")) + + rootCmd.Flags().Bool("leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + _ = viper.BindPFlag("leader-elect", rootCmd.Flags().Lookup("leader-elect")) + + rootCmd.Flags().Bool("metrics-secure", true, + "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + _ = viper.BindPFlag("metrics-secure", rootCmd.Flags().Lookup("metrics-secure")) + + rootCmd.Flags().Bool("enable-http2", false, + "If set, HTTP/2 will be enabled for the metrics and webhook servers") + _ = viper.BindPFlag("enable-http2", rootCmd.Flags().Lookup("enable-http2")) + + rootCmd.Flags().String( + "plugin-path", + "", + "The plugins socket path", + ) + _ = viper.BindPFlag("plugin-path", rootCmd.Flags().Lookup("plugin-path")) + + rootCmd.Flags().String( + "server-cert", + "", + "The public key to be used for the server process", + ) + _ = viper.BindPFlag("server-cert", rootCmd.Flags().Lookup("server-cert")) + + rootCmd.Flags().String( + "server-key", + "", + "The key to be used for the server process", + ) + _ = viper.BindPFlag("server-key", rootCmd.Flags().Lookup("server-key")) + + rootCmd.Flags().String( + "client-cert", + "", + "The client public key to verify the connection", + ) + _ = viper.BindPFlag("client-cert", rootCmd.Flags().Lookup("client-cert")) + + rootCmd.Flags().String( + "server-address", + "", + "The address where to listen (i.e. 0:9090)", + ) + _ = viper.BindPFlag("server-address", rootCmd.Flags().Lookup("server-address")) + + _ = viper.BindEnv("sidecar-image", "SIDECAR_IMAGE") if err := rootCmd.Execute(); err != nil { fmt.Println(err) os.Exit(1) } } - -func newOperatorCommand() *cobra.Command { - cmd := operator.NewCommand() - cmd.Use = "operator" - cmd.Short = "Starts the BarmanObjectStore reconciler and the Barman Cloud CNPG-i plugin" - grpcServer := cmd.RunE - - cmd.RunE = func(cmd *cobra.Command, args []string) error { - operatorPool := pool. - New(). - WithContext(cmd.Context()). - WithCancelOnError(). - WithFirstError() - operatorPool.Go(func(ctx context.Context) error { - cmd.SetContext(ctx) - - if len(viper.GetString("sidecar-image")) == 0 { - return fmt.Errorf("missing required SIDECAR_IMAGE environment variable") - } - - err := grpcServer(cmd, args) - return err - }) - operatorPool.Go(manager.Start) - return operatorPool.Wait() - } - - cmd.Flags().String("metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ - "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") - _ = viper.BindPFlag("metrics-bind-address", cmd.Flags().Lookup("metrics-bind-address")) - - cmd.Flags().String("health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") - _ = viper.BindPFlag("health-probe-bind-address", cmd.Flags().Lookup("health-probe-bind-address")) - - cmd.Flags().Bool("leader-elect", false, - "Enable leader election for controller manager. "+ - "Enabling this will ensure there is only one active controller manager.") - _ = viper.BindPFlag("leader-elect", cmd.Flags().Lookup("leader-elect")) - - cmd.Flags().Bool("metrics-secure", true, - "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") - _ = viper.BindPFlag("metrics-secure", cmd.Flags().Lookup("metrics-secure")) - - cmd.Flags().Bool("enable-http2", false, - "If set, HTTP/2 will be enabled for the metrics and webhook servers") - _ = viper.BindPFlag("enable-http2", cmd.Flags().Lookup("enable-http2")) - - _ = viper.BindEnv("sidecar-image", "SIDECAR_IMAGE") - - return cmd -} diff --git a/config/crd/bases/barmancloud.cnpg.io_objectstores.yaml b/config/crd/bases/barmancloud.cnpg.io_objectstores.yaml index 63f8c2a..5e0d8cd 100644 --- a/config/crd/bases/barmancloud.cnpg.io_objectstores.yaml +++ b/config/crd/bases/barmancloud.cnpg.io_objectstores.yaml @@ -17,7 +17,7 @@ spec: - name: v1 schema: openAPIV3Schema: - description: ObjectStore is the Schema for the objectstores API + description: ObjectStore is the Schema for the objectstores API. properties: apiVersion: description: |- @@ -37,7 +37,7 @@ spec: metadata: type: object spec: - description: ObjectStoreSpec defines the desired state of ObjectStore + description: ObjectStoreSpec defines the desired state of ObjectStore. properties: configuration: description: |- @@ -382,7 +382,7 @@ spec: - configuration type: object status: - description: ObjectStoreStatus defines the observed state of ObjectStore + description: ObjectStoreStatus defines the observed state of ObjectStore. type: object type: object served: true diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 4ade04d..78bfda0 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -2,7 +2,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: manager-role + name: plugin-barman-cloud rules: - apiGroups: - barmancloud.cnpg.io @@ -30,3 +30,15 @@ rules: - get - patch - update +- apiGroups: + - rbac.authorization.k8s.io + resources: + - rolebindings + - roles + verbs: + - create + - get + - list + - patch + - update + - watch diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml index 99e9a5e..3e03bda 100644 --- a/config/rbac/role_binding.yaml +++ b/config/rbac/role_binding.yaml @@ -4,11 +4,11 @@ metadata: labels: app.kubernetes.io/name: plugin-barman-cloud app.kubernetes.io/managed-by: kustomize - name: manager-rolebinding + name: plugin-barman-cloud-binding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: manager-role + name: plugin-barman-cloud subjects: - kind: ServiceAccount name: plugin-barman-cloud diff --git a/config/samples/barmancloud_v1_objectstore.yaml b/config/samples/barmancloud_v1_objectstore.yaml deleted file mode 100644 index 70e982f..0000000 --- a/config/samples/barmancloud_v1_objectstore.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: barmancloud.cnpg.io/v1 -kind: ObjectStore -metadata: - labels: - app.kubernetes.io/name: plugin-barman-cloud - app.kubernetes.io/managed-by: kustomize - name: objectstore-sample -spec: - # TODO(user): Add fields here diff --git a/docs/examples/cluster-example.yaml b/docs/examples/cluster-example.yaml index 33983b8..de6c774 100644 --- a/docs/examples/cluster-example.yaml +++ b/docs/examples/cluster-example.yaml @@ -8,7 +8,7 @@ spec: plugins: - name: barman-cloud.cloudnative-pg.io parameters: - barmanObjectStore: minio-store + barmanObjectName: minio-store storage: size: 1Gi diff --git a/docs/examples/minio-store.yaml b/docs/examples/minio-store.yaml new file mode 100644 index 0000000..0193713 --- /dev/null +++ b/docs/examples/minio-store.yaml @@ -0,0 +1,23 @@ +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: minio-store +spec: + configuration: + destinationPath: s3://backups/ + endpointURL: http://minio:9000 + s3Credentials: + accessKeyId: + name: minio + key: ACCESS_KEY_ID + secretAccessKey: + name: minio + key: ACCESS_SECRET_KEY + wal: + compression: gzip + data: + additionalCommandArgs: + - "--min-chunk-size=5MB" + - "--read-timeout=60" + - "-vv" + diff --git a/docs/minio/minio-client.yaml b/docs/minio/minio-client.yaml new file mode 100644 index 0000000..4d8ebb8 --- /dev/null +++ b/docs/minio/minio-client.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + run: mc + name: mc +spec: + containers: + - env: + - name: MC_HOST_minio + value: http://chooJeiroroo2noquomei2uuceisheth:ongeiqueitohL0queeLohkiur2quaing@minio:9000 + image: minio/mc + name: mc + resources: {} + # Keep the pod up to exec stuff on it + command: + - sleep + - "3600" + dnsPolicy: ClusterFirst + restartPolicy: Always diff --git a/docs/minio/minio-delete.sh b/docs/minio/minio-delete.sh new file mode 100644 index 0000000..c098b3c --- /dev/null +++ b/docs/minio/minio-delete.sh @@ -0,0 +1 @@ +kubectl exec -ti mc -- mc rm -r --force minio/backups diff --git a/docs/minio/minio-deployment.yaml b/docs/minio/minio-deployment.yaml new file mode 100644 index 0000000..315ea6e --- /dev/null +++ b/docs/minio/minio-deployment.yaml @@ -0,0 +1,42 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio + labels: + app: minio +spec: + replicas: 1 + selector: + matchLabels: + app: minio + template: + metadata: + labels: + app: minio + spec: + containers: + - name: minio + image: minio/minio + ports: + - containerPort: 9000 + volumeMounts: + - mountPath: /data + name: data + args: + - server + - /data + env: + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio + key: ACCESS_KEY_ID + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: minio + key: ACCESS_SECRET_KEY + volumes: + - name: data + persistentVolumeClaim: + claimName: minio diff --git a/docs/minio/minio-pvc.yaml b/docs/minio/minio-pvc.yaml new file mode 100644 index 0000000..6a402cf --- /dev/null +++ b/docs/minio/minio-pvc.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: minio +spec: + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + resources: + requests: + storage: 1Gi diff --git a/docs/minio/minio-secret.yaml b/docs/minio/minio-secret.yaml new file mode 100644 index 0000000..0189d9a --- /dev/null +++ b/docs/minio/minio-secret.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +data: + ACCESS_KEY_ID: Y2hvb0plaXJvcm9vMm5vcXVvbWVpMnV1Y2Vpc2hldGg= + ACCESS_SECRET_KEY: b25nZWlxdWVpdG9oTDBxdWVlTG9oa2l1cjJxdWFpbmc= +kind: Secret +metadata: + name: minio diff --git a/docs/minio/minio-service.yaml b/docs/minio/minio-service.yaml new file mode 100644 index 0000000..c2b356b --- /dev/null +++ b/docs/minio/minio-service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: minio +spec: + selector: + app: minio + ports: + - protocol: TCP + port: 9000 + targetPort: 9000 diff --git a/go.mod b/go.mod index 732001d..01a96c5 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,6 @@ require ( github.com/cloudnative-pg/machinery v0.0.0-20241001075747-34c8797af80f github.com/onsi/ginkgo/v2 v2.20.2 github.com/onsi/gomega v1.34.2 - github.com/sourcegraph/conc v0.3.0 github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 google.golang.org/grpc v1.67.1 @@ -80,6 +79,7 @@ require ( github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/snorwin/jsonpatch v1.5.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect diff --git a/internal/cnpgi/operator/config/config.go b/internal/cnpgi/operator/config/config.go new file mode 100644 index 0000000..830e52e --- /dev/null +++ b/internal/cnpgi/operator/config/config.go @@ -0,0 +1,71 @@ +package config + +import ( + "strings" + + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + "github.com/cloudnative-pg/cnpg-i-machinery/pkg/pluginhelper/common" + + "github.com/cloudnative-pg/plugin-barman-cloud/internal/cnpgi/metadata" +) + +// ConfigurationError represents a mistake in the plugin configuration +type ConfigurationError struct { + messages []string +} + +// Error implements the error interface +func (e *ConfigurationError) Error() string { + return strings.Join(e.messages, ",") +} + +// NewConfigurationError creates a new empty configuration error +func NewConfigurationError() *ConfigurationError { + return &ConfigurationError{} +} + +// WithMessage adds a new error message to a potentially empty +// ConfigurationError +func (e *ConfigurationError) WithMessage(msg string) *ConfigurationError { + if e == nil { + return &ConfigurationError{ + messages: []string{msg}, + } + } + + return &ConfigurationError{ + messages: append(e.messages, msg), + } +} + +// IsEmpty returns true if there's no error messages +func (e *ConfigurationError) IsEmpty() bool { + return len(e.messages) == 0 +} + +// PluginConfiguration is the configuration of the plugin +type PluginConfiguration struct { + BarmanObjectName string +} + +// NewFromCluster extracts the configuration from the cluster +func NewFromCluster(cluster *cnpgv1.Cluster) (*PluginConfiguration, error) { + helper := common.NewPlugin( + *cluster, + metadata.PluginName, + ) + + result := &PluginConfiguration{ + BarmanObjectName: helper.Parameters["barmanObjectName"], + } + + err := NewConfigurationError() + if len(result.BarmanObjectName) == 0 { + err = err.WithMessage("Missing barmanObjectName parameter") + } + + if err.IsEmpty() { + return result, nil + } + return result, err +} diff --git a/internal/cnpgi/operator/config/doc.go b/internal/cnpgi/operator/config/doc.go new file mode 100644 index 0000000..703d477 --- /dev/null +++ b/internal/cnpgi/operator/config/doc.go @@ -0,0 +1,2 @@ +// Package config contains the functions to parse the plugin configuration +package config diff --git a/internal/cnpgi/operator/lifecycle.go b/internal/cnpgi/operator/lifecycle.go index 8781cdd..3c44eb7 100644 --- a/internal/cnpgi/operator/lifecycle.go +++ b/internal/cnpgi/operator/lifecycle.go @@ -4,7 +4,6 @@ import ( "context" "errors" - "github.com/cloudnative-pg/cnpg-i-machinery/pkg/pluginhelper/common" "github.com/cloudnative-pg/cnpg-i-machinery/pkg/pluginhelper/decoder" "github.com/cloudnative-pg/cnpg-i-machinery/pkg/pluginhelper/object" "github.com/cloudnative-pg/cnpg-i/pkg/lifecycle" @@ -12,7 +11,7 @@ import ( "github.com/spf13/viper" corev1 "k8s.io/api/core/v1" - "github.com/cloudnative-pg/plugin-barman-cloud/internal/cnpgi/metadata" + "github.com/cloudnative-pg/plugin-barman-cloud/internal/cnpgi/operator/config" ) // LifecycleImplementation is the implementation of the lifecycle handler @@ -65,12 +64,10 @@ func (impl LifecycleImplementation) LifecycleHook( return nil, err } - helper := common.NewPlugin( - *cluster, - metadata.PluginName, - ) - - // TODO: Validation of the plugin configuration + pluginConfiguration, err := config.NewFromCluster(cluster) + if err != nil { + return nil, err + } mutatedPod := pod.DeepCopy() err = object.InjectPluginSidecar(mutatedPod, &corev1.Container{ @@ -79,7 +76,7 @@ func (impl LifecycleImplementation) LifecycleHook( Env: []corev1.EnvVar{ { Name: "BARMAN_OBJECT_NAME", - Value: helper.Parameters["barmanObjectStore"], + Value: pluginConfiguration.BarmanObjectName, }, { // TODO: should we really use this one? diff --git a/internal/operator/manager/manager.go b/internal/cnpgi/operator/manager.go similarity index 90% rename from internal/operator/manager/manager.go rename to internal/cnpgi/operator/manager.go index ad37848..f7ca78a 100644 --- a/internal/operator/manager/manager.go +++ b/internal/cnpgi/operator/manager.go @@ -14,15 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package manager contains the implementation of the ObjectStore controller manager -package manager +package operator import ( "context" "crypto/tls" - "flag" // +kubebuilder:scaffold:imports + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" "github.com/cloudnative-pg/machinery/pkg/log" "github.com/spf13/viper" "k8s.io/apimachinery/pkg/runtime" @@ -30,13 +29,12 @@ import ( clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" - "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" barmancloudv1 "github.com/cloudnative-pg/plugin-barman-cloud/api/v1" - "github.com/cloudnative-pg/plugin-barman-cloud/internal/operator/controller" + "github.com/cloudnative-pg/plugin-barman-cloud/internal/controller" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -47,8 +45,8 @@ var scheme = runtime.NewScheme() func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) - utilruntime.Must(barmancloudv1.AddToScheme(scheme)) + utilruntime.Must(cnpgv1.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } @@ -58,14 +56,6 @@ func Start(ctx context.Context) error { var tlsOpts []func(*tls.Config) - opts := zap.Options{ - Development: true, - } - opts.BindFlags(flag.CommandLine) - flag.Parse() - - ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) - // if the enable-http2 flag is false (the default), http/2 should be disabled // due to its vulnerabilities. More specifically, disabling http/2 will // prevent from being vulnerable to the HTTP/2 Stream Cancellation and @@ -151,6 +141,18 @@ func Start(ctx context.Context) error { return err } + if err := mgr.Add(&CNPGI{ + Client: mgr.GetClient(), + PluginPath: viper.GetString("plugin-path"), + ServerCertPath: viper.GetString("server-cert"), + ServerKeyPath: viper.GetString("server-key"), + ClientCertPath: viper.GetString("client-cert"), + ServerAddress: viper.GetString("server-address"), + }); err != nil { + setupLog.Error(err, "unable to create CNPGI runnable") + return err + } + setupLog.Info("starting manager") if err := mgr.Start(ctx); err != nil { setupLog.Error(err, "problem running manager") diff --git a/internal/cnpgi/operator/reconciler.go b/internal/cnpgi/operator/reconciler.go index b0e3063..9fabee8 100644 --- a/internal/cnpgi/operator/reconciler.go +++ b/internal/cnpgi/operator/reconciler.go @@ -2,12 +2,24 @@ package operator import ( "context" + "fmt" + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + "github.com/cloudnative-pg/cnpg-i-machinery/pkg/pluginhelper/decoder" + "github.com/cloudnative-pg/cnpg-i-machinery/pkg/pluginhelper/object" "github.com/cloudnative-pg/cnpg-i/pkg/reconciler" + rbacv1 "k8s.io/api/rbac/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/cloudnative-pg/plugin-barman-cloud/internal/cnpgi/operator/config" ) // ReconcilerImplementation implements the Reconciler capability type ReconcilerImplementation struct { + Client client.Client reconciler.UnimplementedReconcilerHooksServer } @@ -30,9 +42,37 @@ func (r ReconcilerImplementation) GetCapabilities( // Pre implements the reconciler interface func (r ReconcilerImplementation) Pre( - _ context.Context, - _ *reconciler.ReconcilerHooksRequest, + ctx context.Context, + request *reconciler.ReconcilerHooksRequest, ) (*reconciler.ReconcilerHooksResult, error) { + reconciledKind, err := object.GetKind(request.GetResourceDefinition()) + if err != nil { + return nil, err + } + if reconciledKind != "Cluster" { + return &reconciler.ReconcilerHooksResult{ + Behavior: reconciler.ReconcilerHooksResult_BEHAVIOR_CONTINUE, + }, nil + } + + cluster, err := decoder.DecodeClusterJSON(request.GetResourceDefinition()) + if err != nil { + return nil, err + } + + pluginConfiguration, err := config.NewFromCluster(cluster) + if err != nil { + return nil, err + } + + if err := r.ensureRole(ctx, cluster, pluginConfiguration.BarmanObjectName); err != nil { + return nil, err + } + + if err := r.ensureRoleBinding(ctx, cluster); err != nil { + return nil, err + } + return &reconciler.ReconcilerHooksResult{ Behavior: reconciler.ReconcilerHooksResult_BEHAVIOR_CONTINUE, }, nil @@ -47,3 +87,126 @@ func (r ReconcilerImplementation) Post( Behavior: reconciler.ReconcilerHooksResult_BEHAVIOR_CONTINUE, }, nil } + +func (r ReconcilerImplementation) ensureRole( + ctx context.Context, + cluster *cnpgv1.Cluster, + barmanObjectName string, +) error { + var role rbacv1.Role + if err := r.Client.Get(ctx, client.ObjectKey{ + Namespace: cluster.Namespace, + Name: getRBACName(cluster.Name), + }, &role); err != nil { + if apierrs.IsNotFound(err) { + return r.createRole(ctx, cluster, barmanObjectName) + } + return err + } + + // TODO: patch existing role + return nil +} + +func (r ReconcilerImplementation) ensureRoleBinding( + ctx context.Context, + cluster *cnpgv1.Cluster, +) error { + var role rbacv1.RoleBinding + if err := r.Client.Get(ctx, client.ObjectKey{ + Namespace: cluster.Namespace, + Name: getRBACName(cluster.Name), + }, &role); err != nil { + if apierrs.IsNotFound(err) { + return r.createRoleBinding(ctx, cluster) + } + return err + } + + // TODO: patch existing role binding + return nil +} + +func (r ReconcilerImplementation) createRole( + ctx context.Context, + cluster *cnpgv1.Cluster, + barmanObjectName string, +) error { + role := buildRole(cluster, barmanObjectName) + if err := ctrl.SetControllerReference(cluster, role, r.Client.Scheme()); err != nil { + return err + } + return r.Client.Create(ctx, role) +} + +func (r ReconcilerImplementation) createRoleBinding( + ctx context.Context, + cluster *cnpgv1.Cluster, +) error { + roleBinding := buildRoleBinding(cluster) + if err := ctrl.SetControllerReference(cluster, roleBinding, r.Client.Scheme()); err != nil { + return err + } + return r.Client.Create(ctx, roleBinding) +} + +func buildRole( + cluster *cnpgv1.Cluster, + barmanObjectName string, +) *rbacv1.Role { + return &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: cluster.Namespace, + Name: getRBACName(cluster.Name), + }, + + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{ + "barmancloud.cnpg.io", + }, + Verbs: []string{ + "get", + "watch", + "list", + }, + Resources: []string{ + "objectstores", + }, + ResourceNames: []string{ + barmanObjectName, + }, + }, + }, + } +} + +func buildRoleBinding( + cluster *cnpgv1.Cluster, +) *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: cluster.Namespace, + Name: getRBACName(cluster.Name), + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + APIGroup: "", + Name: cluster.Name, + Namespace: cluster.Namespace, + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: getRBACName(cluster.Name), + }, + } +} + +// getRBACName returns the name of the RBAC entities for the +// barman cloud plugin +func getRBACName(clusterName string) string { + return fmt.Sprintf("%s-barman", clusterName) +} diff --git a/internal/cnpgi/operator/start.go b/internal/cnpgi/operator/start.go index 21cd29c..3c8eff6 100644 --- a/internal/cnpgi/operator/start.go +++ b/internal/cnpgi/operator/start.go @@ -1,21 +1,45 @@ package operator import ( + "context" + "github.com/cloudnative-pg/cnpg-i-machinery/pkg/pluginhelper/http" "github.com/cloudnative-pg/cnpg-i/pkg/lifecycle" "github.com/cloudnative-pg/cnpg-i/pkg/reconciler" - "github.com/spf13/cobra" "google.golang.org/grpc" + "sigs.k8s.io/controller-runtime/pkg/client" ) -// NewCommand creates the command to start the GRPC server +// CNPGI is the implementation of the CNPG-i server +type CNPGI struct { + Client client.Client + PluginPath string + ServerCertPath string + ServerKeyPath string + ClientCertPath string + ServerAddress string +} + +// Start starts the GRPC server // of the operator plugin -func NewCommand() *cobra.Command { - cmd := http.CreateMainCmd(IdentityImplementation{}, func(server *grpc.Server) error { - reconciler.RegisterReconcilerHooksServer(server, ReconcilerImplementation{}) +func (c *CNPGI) Start(ctx context.Context) error { + enrich := func(server *grpc.Server) error { + reconciler.RegisterReconcilerHooksServer(server, ReconcilerImplementation{ + Client: c.Client, + }) lifecycle.RegisterOperatorLifecycleServer(server, LifecycleImplementation{}) return nil - }) - cmd.Use = "plugin" - return cmd + } + + srv := http.Server{ + IdentityImpl: IdentityImplementation{}, + Enrichers: []http.ServerEnricher{enrich}, + PluginPath: c.PluginPath, + ServerCertPath: c.ServerCertPath, + ServerKeyPath: c.ServerKeyPath, + ClientCertPath: c.ClientCertPath, + ServerAddress: c.ServerAddress, + } + + return srv.Start(ctx) } diff --git a/internal/operator/controller/doc.go b/internal/controller/doc.go similarity index 100% rename from internal/operator/controller/doc.go rename to internal/controller/doc.go diff --git a/internal/operator/controller/objectstore_controller.go b/internal/controller/objectstore_controller.go similarity index 91% rename from internal/operator/controller/objectstore_controller.go rename to internal/controller/objectstore_controller.go index 6f24a1a..b16a5c4 100644 --- a/internal/operator/controller/objectstore_controller.go +++ b/internal/controller/objectstore_controller.go @@ -34,6 +34,8 @@ type ObjectStoreReconciler struct { Scheme *runtime.Scheme } +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=rolebindings,verbs=create;patch;update;get;list;watch +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles,verbs=create;patch;update;get;list;watch // +kubebuilder:rbac:groups=barmancloud.cnpg.io,resources=objectstores,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=barmancloud.cnpg.io,resources=objectstores/status,verbs=get;update;patch // +kubebuilder:rbac:groups=barmancloud.cnpg.io,resources=objectstores/finalizers,verbs=update diff --git a/internal/operator/controller/objectstore_controller_test.go b/internal/controller/objectstore_controller_test.go similarity index 100% rename from internal/operator/controller/objectstore_controller_test.go rename to internal/controller/objectstore_controller_test.go diff --git a/internal/operator/controller/suite_test.go b/internal/controller/suite_test.go similarity index 96% rename from internal/operator/controller/suite_test.go rename to internal/controller/suite_test.go index 2cbe841..d58ef20 100644 --- a/internal/operator/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -61,7 +61,7 @@ var _ = BeforeSuite(func() { By("bootstrapping test environment") testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")}, + CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, ErrorIfCRDPathMissing: true, // The BinaryAssetsDirectory is only required if you want to run the tests directly diff --git a/kubernetes/deployment.yaml b/kubernetes/deployment.yaml index 29820bb..61900f7 100644 --- a/kubernetes/deployment.yaml +++ b/kubernetes/deployment.yaml @@ -29,7 +29,6 @@ spec: key: SIDECAR_IMAGE name: plugin-barman-cloud args: - - operator - --server-cert=/server/tls.crt - --server-key=/server/tls.key - --client-cert=/client/tls.crt diff --git a/kubernetes/kustomization.yaml b/kubernetes/kustomization.yaml index 31d8603..94f9b5d 100644 --- a/kubernetes/kustomization.yaml +++ b/kubernetes/kustomization.yaml @@ -12,8 +12,8 @@ resources: images: - name: plugin-barman-cloud newName: kind.local/github.com/cloudnative-pg/plugin-barman-cloud/cmd/operator - newTag: 634ab82d4f8b68ffada6033b7dff817bbed61a2fec8e05f9ebf74f5bedafb0dd + newTag: 7e901b38eaf33b047dcf2eb044c9c8ca85535d8041a3144d25f7e1a4690ea071 secretGenerator: - literals: - - SIDECAR_IMAGE=kind.local/github.com/cloudnative-pg/plugin-barman-cloud/cmd/instance:7173a2dcf3ce74e982ca3d35053de40fcec67b31d607a0d12547b4f9d09c535c + - SIDECAR_IMAGE=kind.local/github.com/cloudnative-pg/plugin-barman-cloud/cmd/instance:ca1fd58413940a247bc52cdb44f4a6909192d781b1767dc7ee9625368ee9d7e2 name: plugin-barman-cloud From e78aee07da478af5b1a8ea5e7b9286bcff46ef34 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 2 Oct 2024 15:03:21 +0200 Subject: [PATCH 3/4] chore(deps): update golang docker tag to v1.23.2 (#21) | datasource | package | from | to | | ---------- | ------- | ------ | ------ | | docker | golang | 1.23.1 | 1.23.2 | Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- Taskfile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Taskfile.yml b/Taskfile.yml index 3623e9d..8e25d35 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -51,7 +51,7 @@ tasks: desc: Run go test env: # renovate: datasource=docker depName=golang versioning=semver - GOLANG_IMAGE_VERSION: 1.23.1 + GOLANG_IMAGE_VERSION: 1.23.2 # renovate: datasource=git-refs depname=kubernetes packageName=https://github.com/kubernetes/kubernetes versioning=semver K8S_VERSION: 1.31.0 # renovate: datasource=git-refs depName=controller-runtime packageName=https://github.com/kubernetes-sigs/controller-runtime versioning=semver From 76383a30afd3bd829f01936dc3dfc81f1d189d2d Mon Sep 17 00:00:00 2001 From: Leonardo Cecchi Date: Thu, 3 Oct 2024 16:58:56 +0200 Subject: [PATCH 4/4] feat: grant permissions to read secrets (#25) Signed-off-by: Leonardo Cecchi --- config/rbac/role.yaml | 10 ++ internal/cnpgi/instance/manager.go | 10 ++ internal/cnpgi/instance/wal.go | 23 ++- internal/cnpgi/operator/reconciler.go | 151 ++++++++---------- internal/cnpgi/operator/specs/role.go | 88 ++++++++++ internal/cnpgi/operator/specs/secrets.go | 51 ++++++ internal/cnpgi/operator/specs/specs.go | 3 + internal/controller/objectstore_controller.go | 1 + scripts/run.sh | 2 +- 9 files changed, 248 insertions(+), 91 deletions(-) create mode 100644 internal/cnpgi/operator/specs/role.go create mode 100644 internal/cnpgi/operator/specs/secrets.go create mode 100644 internal/cnpgi/operator/specs/specs.go diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 78bfda0..59a7677 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -4,6 +4,16 @@ kind: ClusterRole metadata: name: plugin-barman-cloud rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - get + - list + - watch - apiGroups: - barmancloud.cnpg.io resources: diff --git a/internal/cnpgi/instance/manager.go b/internal/cnpgi/instance/manager.go index 18e8ff4..5b2072b 100644 --- a/internal/cnpgi/instance/manager.go +++ b/internal/cnpgi/instance/manager.go @@ -7,9 +7,11 @@ import ( cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" "github.com/spf13/viper" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" @@ -23,6 +25,7 @@ var scheme = runtime.NewScheme() func init() { utilruntime.Must(barmancloudv1.AddToScheme(scheme)) utilruntime.Must(cnpgv1.AddToScheme(scheme)) + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) } // Start starts the sidecar informers and CNPG-i server @@ -52,6 +55,13 @@ func Start(ctx context.Context) error { }, }, }, + Client: client.Options{ + Cache: &client.CacheOptions{ + DisableFor: []client.Object{ + &corev1.Secret{}, + }, + }, + }, }) if err != nil { setupLog.Error(err, "unable to start manager") diff --git a/internal/cnpgi/instance/wal.go b/internal/cnpgi/instance/wal.go index fec6dbd..25eb19c 100644 --- a/internal/cnpgi/instance/wal.go +++ b/internal/cnpgi/instance/wal.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "os" + "path" "strings" "time" @@ -22,6 +23,13 @@ import ( barmancloudv1 "github.com/cloudnative-pg/plugin-barman-cloud/api/v1" ) +const ( + // CheckEmptyWalArchiveFile is the name of the file in the PGDATA that, + // if present, requires the WAL archiver to check that the backup object + // store is empty. + CheckEmptyWalArchiveFile = ".check-empty-wal-archive" +) + // WALServiceImplementation is the implementation of the WAL Service type WALServiceImplementation struct { BarmanObjectKey client.ObjectKey @@ -75,11 +83,20 @@ func (w WALServiceImplementation) Archive( objectStore.Namespace, &objectStore.Spec.Configuration, os.Environ()) - if apierrors.IsForbidden(err) { - return nil, errors.New("backup credentials don't yet have access permissions. Will retry reconciliation loop") + if err != nil { + if apierrors.IsForbidden(err) { + return nil, errors.New("backup credentials don't yet have access permissions. Will retry reconciliation loop") + } + return nil, err } - arch, err := archiver.New(ctx, envArchive, w.SpoolDirectory, w.PGDataPath, w.PGWALPath) + arch, err := archiver.New( + ctx, + envArchive, + w.SpoolDirectory, + w.PGDataPath, + path.Join(w.PGDataPath, CheckEmptyWalArchiveFile), + ) if err != nil { return nil, err } diff --git a/internal/cnpgi/operator/reconciler.go b/internal/cnpgi/operator/reconciler.go index 9fabee8..3ce4147 100644 --- a/internal/cnpgi/operator/reconciler.go +++ b/internal/cnpgi/operator/reconciler.go @@ -2,19 +2,21 @@ package operator import ( "context" - "fmt" cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" "github.com/cloudnative-pg/cnpg-i-machinery/pkg/pluginhelper/decoder" "github.com/cloudnative-pg/cnpg-i-machinery/pkg/pluginhelper/object" "github.com/cloudnative-pg/cnpg-i/pkg/reconciler" rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/equality" apierrs "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + barmancloudv1 "github.com/cloudnative-pg/plugin-barman-cloud/api/v1" "github.com/cloudnative-pg/plugin-barman-cloud/internal/cnpgi/operator/config" + "github.com/cloudnative-pg/plugin-barman-cloud/internal/cnpgi/operator/specs" ) // ReconcilerImplementation implements the Reconciler capability @@ -45,6 +47,8 @@ func (r ReconcilerImplementation) Pre( ctx context.Context, request *reconciler.ReconcilerHooksRequest, ) (*reconciler.ReconcilerHooksResult, error) { + contextLogger := log.FromContext(ctx) + reconciledKind, err := object.GetKind(request.GetResourceDefinition()) if err != nil { return nil, err @@ -60,12 +64,28 @@ func (r ReconcilerImplementation) Pre( return nil, err } + contextLogger = contextLogger.WithValues("name", cluster.Name, "namespace", cluster.Namespace) + ctx = log.IntoContext(ctx, contextLogger) + pluginConfiguration, err := config.NewFromCluster(cluster) if err != nil { return nil, err } - if err := r.ensureRole(ctx, cluster, pluginConfiguration.BarmanObjectName); err != nil { + var barmanObject barmancloudv1.ObjectStore + if err := r.Client.Get(ctx, client.ObjectKey{ + Namespace: cluster.Namespace, + Name: pluginConfiguration.BarmanObjectName, + }, &barmanObject); err != nil { + if apierrs.IsNotFound(err) { + contextLogger.Info("Not found barman object configuration, requeuing") + return &reconciler.ReconcilerHooksResult{ + Behavior: reconciler.ReconcilerHooksResult_BEHAVIOR_REQUEUE, + }, nil + } + } + + if err := r.ensureRole(ctx, cluster, &barmanObject); err != nil { return nil, err } @@ -91,21 +111,50 @@ func (r ReconcilerImplementation) Post( func (r ReconcilerImplementation) ensureRole( ctx context.Context, cluster *cnpgv1.Cluster, - barmanObjectName string, + barmanObject *barmancloudv1.ObjectStore, ) error { + contextLogger := log.FromContext(ctx) + newRole := specs.BuildRole(cluster, barmanObject) + var role rbacv1.Role if err := r.Client.Get(ctx, client.ObjectKey{ - Namespace: cluster.Namespace, - Name: getRBACName(cluster.Name), + Namespace: newRole.Namespace, + Name: newRole.Name, }, &role); err != nil { - if apierrs.IsNotFound(err) { - return r.createRole(ctx, cluster, barmanObjectName) + if !apierrs.IsNotFound(err) { + return err } - return err + + contextLogger.Info( + "Creating role", + "name", newRole.Name, + "namespace", newRole.Namespace, + ) + + if err := ctrl.SetControllerReference( + cluster, + newRole, + r.Client.Scheme(), + ); err != nil { + return err + } + + return r.Client.Create(ctx, newRole) } - // TODO: patch existing role - return nil + if equality.Semantic.DeepEqual(newRole.Rules, role.Rules) { + // There's no need to hit the API server again + return nil + } + + contextLogger.Info( + "Patching role", + "name", newRole.Name, + "namespace", newRole.Namespace, + "rules", newRole.Rules, + ) + + return r.Client.Patch(ctx, newRole, client.MergeFrom(&role)) } func (r ReconcilerImplementation) ensureRoleBinding( @@ -115,7 +164,7 @@ func (r ReconcilerImplementation) ensureRoleBinding( var role rbacv1.RoleBinding if err := r.Client.Get(ctx, client.ObjectKey{ Namespace: cluster.Namespace, - Name: getRBACName(cluster.Name), + Name: specs.GetRBACName(cluster.Name), }, &role); err != nil { if apierrs.IsNotFound(err) { return r.createRoleBinding(ctx, cluster) @@ -123,90 +172,18 @@ func (r ReconcilerImplementation) ensureRoleBinding( return err } - // TODO: patch existing role binding + // TODO: this assumes role bindings never change. + // Is that true? Should we relax this assumption? return nil } -func (r ReconcilerImplementation) createRole( - ctx context.Context, - cluster *cnpgv1.Cluster, - barmanObjectName string, -) error { - role := buildRole(cluster, barmanObjectName) - if err := ctrl.SetControllerReference(cluster, role, r.Client.Scheme()); err != nil { - return err - } - return r.Client.Create(ctx, role) -} - func (r ReconcilerImplementation) createRoleBinding( ctx context.Context, cluster *cnpgv1.Cluster, ) error { - roleBinding := buildRoleBinding(cluster) + roleBinding := specs.BuildRoleBinding(cluster) if err := ctrl.SetControllerReference(cluster, roleBinding, r.Client.Scheme()); err != nil { return err } return r.Client.Create(ctx, roleBinding) } - -func buildRole( - cluster *cnpgv1.Cluster, - barmanObjectName string, -) *rbacv1.Role { - return &rbacv1.Role{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: cluster.Namespace, - Name: getRBACName(cluster.Name), - }, - - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{ - "barmancloud.cnpg.io", - }, - Verbs: []string{ - "get", - "watch", - "list", - }, - Resources: []string{ - "objectstores", - }, - ResourceNames: []string{ - barmanObjectName, - }, - }, - }, - } -} - -func buildRoleBinding( - cluster *cnpgv1.Cluster, -) *rbacv1.RoleBinding { - return &rbacv1.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: cluster.Namespace, - Name: getRBACName(cluster.Name), - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - APIGroup: "", - Name: cluster.Name, - Namespace: cluster.Namespace, - }, - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "Role", - Name: getRBACName(cluster.Name), - }, - } -} - -// getRBACName returns the name of the RBAC entities for the -// barman cloud plugin -func getRBACName(clusterName string) string { - return fmt.Sprintf("%s-barman", clusterName) -} diff --git a/internal/cnpgi/operator/specs/role.go b/internal/cnpgi/operator/specs/role.go new file mode 100644 index 0000000..3f2b411 --- /dev/null +++ b/internal/cnpgi/operator/specs/role.go @@ -0,0 +1,88 @@ +package specs + +import ( + "fmt" + + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + barmancloudv1 "github.com/cloudnative-pg/plugin-barman-cloud/api/v1" +) + +// BuildRole builds the Role object for this cluster +func BuildRole( + cluster *cnpgv1.Cluster, + barmanObject *barmancloudv1.ObjectStore, +) *rbacv1.Role { + return &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: cluster.Namespace, + Name: GetRBACName(cluster.Name), + }, + + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{ + "barmancloud.cnpg.io", + }, + Verbs: []string{ + "get", + "watch", + "list", + }, + Resources: []string{ + "objectstores", + }, + ResourceNames: []string{ + barmanObject.Name, + }, + }, + { + APIGroups: []string{ + "", + }, + Resources: []string{ + "secrets", + }, + Verbs: []string{ + "get", + "watch", + "list", + }, + ResourceNames: collectSecretNames(barmanObject), + }, + }, + } +} + +// BuildRoleBinding builds the role binding object for this cluster +func BuildRoleBinding( + cluster *cnpgv1.Cluster, +) *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: cluster.Namespace, + Name: GetRBACName(cluster.Name), + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + APIGroup: "", + Name: cluster.Name, + Namespace: cluster.Namespace, + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: GetRBACName(cluster.Name), + }, + } +} + +// GetRBACName returns the name of the RBAC entities for the +// barman cloud plugin +func GetRBACName(clusterName string) string { + return fmt.Sprintf("%s-barman-cloud", clusterName) +} diff --git a/internal/cnpgi/operator/specs/secrets.go b/internal/cnpgi/operator/specs/secrets.go new file mode 100644 index 0000000..a567cf5 --- /dev/null +++ b/internal/cnpgi/operator/specs/secrets.go @@ -0,0 +1,51 @@ +package specs + +import ( + machineryapi "github.com/cloudnative-pg/machinery/pkg/api" + + barmancloudv1 "github.com/cloudnative-pg/plugin-barman-cloud/api/v1" +) + +func collectSecretNames(object *barmancloudv1.ObjectStore) []string { + if object == nil { + return nil + } + + var references []*machineryapi.SecretKeySelector + if object.Spec.Configuration.AWS != nil { + references = append( + references, + object.Spec.Configuration.AWS.AccessKeyIDReference, + object.Spec.Configuration.AWS.SecretAccessKeyReference, + object.Spec.Configuration.AWS.RegionReference, + object.Spec.Configuration.AWS.SessionToken, + ) + } + if object.Spec.Configuration.Azure != nil { + references = append( + references, + object.Spec.Configuration.Azure.ConnectionString, + object.Spec.Configuration.Azure.StorageAccount, + object.Spec.Configuration.Azure.StorageKey, + object.Spec.Configuration.Azure.StorageSasToken, + ) + } + if object.Spec.Configuration.Google != nil { + references = append( + references, + object.Spec.Configuration.Google.ApplicationCredentials, + ) + } + + result := make([]string, 0, len(references)) + for _, reference := range references { + if reference == nil { + continue + } + result = append(result, reference.Name) + } + + // TODO: stringset belongs to machinery :( + + return result +} diff --git a/internal/cnpgi/operator/specs/specs.go b/internal/cnpgi/operator/specs/specs.go new file mode 100644 index 0000000..9b20503 --- /dev/null +++ b/internal/cnpgi/operator/specs/specs.go @@ -0,0 +1,3 @@ +// Package specs contains the specification of the kubernetes objects +// that are created by the plugin +package specs diff --git a/internal/controller/objectstore_controller.go b/internal/controller/objectstore_controller.go index b16a5c4..33c3f4c 100644 --- a/internal/controller/objectstore_controller.go +++ b/internal/controller/objectstore_controller.go @@ -36,6 +36,7 @@ type ObjectStoreReconciler struct { // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=rolebindings,verbs=create;patch;update;get;list;watch // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles,verbs=create;patch;update;get;list;watch +// +kubebuilder:rbac:groups="",resources=secrets,verbs=create;list;get;watch;delete // +kubebuilder:rbac:groups=barmancloud.cnpg.io,resources=objectstores,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=barmancloud.cnpg.io,resources=objectstores/status,verbs=get;update;patch // +kubebuilder:rbac:groups=barmancloud.cnpg.io,resources=objectstores/finalizers,verbs=update diff --git a/scripts/run.sh b/scripts/run.sh index 994c6e9..730bc53 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -10,7 +10,7 @@ fi current_context=$(kubectl config view --raw -o json | jq -r '."current-context"' | sed "s/kind-//") operator_image=$(KIND_CLUSTER_NAME="$current_context" KO_DOCKER_REPO=kind.local ko build -BP ./cmd/operator) -instance_image=$(KIND_CLUSTER_NAME="$current_context" KO_DOCKER_REPO=kind.local ko build -BP ./cmd/instance) +instance_image=$(KIND_CLUSTER_NAME="$current_context" KO_DOCKER_REPO=kind.local KO_DEFAULTBASEIMAGE="ghcr.io/cloudnative-pg/postgresql:17.0" ko build -BP ./cmd/instance) ( cd kubernetes;