From af60a158377c4e29c5d4bd25557ae91c7e04c9bb Mon Sep 17 00:00:00 2001 From: Francesco Canovai Date: Mon, 2 Dec 2024 15:53:34 +0100 Subject: [PATCH] test(e2e): backup and restore (#71) Run basic backup and restore tests for the plugin. Use MinIO for S3, Azurite for ACS and fake-gcs-server for GCS. Signed-off-by: Francesco Canovai --- Taskfile.yml | 2 +- go.mod | 4 +- go.sum | 8 +- internal/cnpgi/operator/lifecycle.go | 2 +- kubernetes/deployment.yaml | 1 - test/e2e/e2e_suite_test.go | 31 +- test/e2e/internal/certmanager/certmanager.go | 14 +- test/e2e/internal/client/client.go | 56 ++ .../{e2e_test.go => internal/client/doc.go} | 16 +- .../internal/cloudnativepg/cloudnativepg.go | 29 +- test/e2e/internal/cluster/cluster.go | 35 + test/e2e/internal/cluster/doc.go | 18 + test/e2e/internal/command/command.go | 86 +++ test/e2e/internal/command/doc.go | 18 + test/e2e/internal/deployment/deployment.go | 7 +- test/e2e/internal/e2etestenv/main.go | 22 +- test/e2e/internal/kind/cluster.go | 42 +- test/e2e/internal/kustomize/kustomize.go | 55 +- test/e2e/internal/namespace/doc.go | 18 + test/e2e/internal/namespace/namespace.go | 53 ++ test/e2e/internal/objectstore/azurite.go | 203 ++++++ test/e2e/internal/objectstore/doc.go | 18 + .../e2e/internal/objectstore/fakegcsserver.go | 196 ++++++ test/e2e/internal/objectstore/minio.go | 225 ++++++ test/e2e/internal/objectstore/objectstore.go | 57 ++ .../internal/tests/backup/backup_restore.go | 211 ++++++ test/e2e/internal/tests/backup/doc.go | 19 + test/e2e/internal/tests/backup/fixtures.go | 656 ++++++++++++++++++ 28 files changed, 2001 insertions(+), 101 deletions(-) create mode 100644 test/e2e/internal/client/client.go rename test/e2e/{e2e_test.go => internal/client/doc.go} (69%) create mode 100644 test/e2e/internal/cluster/cluster.go create mode 100644 test/e2e/internal/cluster/doc.go create mode 100644 test/e2e/internal/command/command.go create mode 100644 test/e2e/internal/command/doc.go create mode 100644 test/e2e/internal/namespace/doc.go create mode 100644 test/e2e/internal/namespace/namespace.go create mode 100644 test/e2e/internal/objectstore/azurite.go create mode 100644 test/e2e/internal/objectstore/doc.go create mode 100644 test/e2e/internal/objectstore/fakegcsserver.go create mode 100644 test/e2e/internal/objectstore/minio.go create mode 100644 test/e2e/internal/objectstore/objectstore.go create mode 100644 test/e2e/internal/tests/backup/backup_restore.go create mode 100644 test/e2e/internal/tests/backup/doc.go create mode 100644 test/e2e/internal/tests/backup/fixtures.go diff --git a/Taskfile.yml b/Taskfile.yml index 0ee8b36..3a3183e 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -198,7 +198,7 @@ tasks: deps: - build-images cmds: - - go test -v ./test/e2e + - go test -timeout 30m -v ./test/e2e ci: desc: Run the CI pipeline diff --git a/go.mod b/go.mod index 2c16033..1924834 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ toolchain go1.23.3 require ( github.com/cert-manager/cert-manager v1.16.2 + github.com/cloudnative-pg/api v0.0.0-20241116094849-219d7a1d257f github.com/cloudnative-pg/barman-cloud v0.0.0-20241105055149-ae6c2408bd14 github.com/cloudnative-pg/cloudnative-pg v1.24.1-0.20241113134512-8608232c2813 github.com/cloudnative-pg/cnpg-i v0.0.0-20241109002750-8abd359df734 @@ -40,7 +41,6 @@ require ( github.com/blang/semver/v4 v4.0.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/cloudnative-pg/api v0.0.0-20241004125129-98baa9f4957b // indirect github.com/containerd/log v0.1.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.6.0 // indirect @@ -78,7 +78,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.9 // indirect + github.com/klauspost/compress v1.17.11 // indirect github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0 // indirect github.com/lib/pq v1.10.9 // indirect github.com/magiconair/properties v1.8.7 // indirect diff --git a/go.sum b/go.sum index 114462d..9c10fc2 100644 --- a/go.sum +++ b/go.sum @@ -24,8 +24,8 @@ github.com/cert-manager/cert-manager v1.16.2 h1:c9UU2E+8XWGruyvC/mdpc1wuLddtgmNr github.com/cert-manager/cert-manager v1.16.2/go.mod h1:MfLVTL45hFZsqmaT1O0+b2ugaNNQQZttSFV9hASHUb0= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cloudnative-pg/api v0.0.0-20241004125129-98baa9f4957b h1:LZ9tIgKmWb8ZvyLg/J8ExXtmBtEWP2dr3Y4TU4nCq/w= -github.com/cloudnative-pg/api v0.0.0-20241004125129-98baa9f4957b/go.mod h1:mzd1EvoLYy16jJdne6/4nwhoj7t4IZ0MqJMEH4mla8Q= +github.com/cloudnative-pg/api v0.0.0-20241116094849-219d7a1d257f h1:KAXst7XLaipdFk9Qv796+tThfEJwFMG4wPLAizZ7wx4= +github.com/cloudnative-pg/api v0.0.0-20241116094849-219d7a1d257f/go.mod h1:aYVZDHteiejVYbntDxJVx1K45xeV8y0KtR/wK4zvt7U= github.com/cloudnative-pg/barman-cloud v0.0.0-20241105055149-ae6c2408bd14 h1:HX5pXyzVAqfjcDgCa1l8b4sumf7XYnGqiP+6XMgbB2E= github.com/cloudnative-pg/barman-cloud v0.0.0-20241105055149-ae6c2408bd14/go.mod h1:HPGwXHlatQEnb2HdsbGTZLEo8qlxKLdxTHiTeF9TTqw= github.com/cloudnative-pg/cloudnative-pg v1.24.1-0.20241113134512-8608232c2813 h1:XWpr5y28JRwcA4BzxBkHFx7C8JDqOSdDIN7RbRdI6Dg= @@ -126,8 +126,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= diff --git a/internal/cnpgi/operator/lifecycle.go b/internal/cnpgi/operator/lifecycle.go index c192967..c7ab18b 100644 --- a/internal/cnpgi/operator/lifecycle.go +++ b/internal/cnpgi/operator/lifecycle.go @@ -239,7 +239,7 @@ func reconcilePodSpec( FailureThreshold: 3, ProbeHandler: corev1.ProbeHandler{ Exec: &corev1.ExecAction{ - Command: []string{"manager", "healthcheck", "unix"}, + Command: []string{"/manager", "healthcheck", "unix"}, }, }, } diff --git a/kubernetes/deployment.yaml b/kubernetes/deployment.yaml index 3d09adb..dd8df11 100644 --- a/kubernetes/deployment.yaml +++ b/kubernetes/deployment.yaml @@ -18,7 +18,6 @@ spec: serviceAccountName: plugin-barman-cloud containers: - image: plugin-barman-cloud:latest - imagePullPolicy: IfNotPresent name: barman-cloud ports: - containerPort: 9090 diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 690778e..72e7ccd 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -23,21 +23,25 @@ import ( "time" certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + cloudnativepgv1 "github.com/cloudnative-pg/api/pkg/api/v1" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/apimachinery/pkg/runtime" apimachineryTypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" kustomizeTypes "sigs.k8s.io/kustomize/api/types" + "sigs.k8s.io/kustomize/kyaml/resid" + pluginBarmanCloudV1 "github.com/cloudnative-pg/plugin-barman-cloud/api/v1" "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/deployment" "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/e2etestenv" "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/kustomize" + _ "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/tests/backup" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -72,9 +76,26 @@ var _ = SynchronizedBeforeSuite(func(ctx SpecContext) []byte { }, }, }, + Patches: []kustomizeTypes.Patch{ + { + Patch: `[{"op": "replace", "path": "/spec/template/spec/containers/0/imagePullPolicy", "value": "Always"}]`, + Target: &kustomizeTypes.Selector{ + ResId: resid.ResId{ + Gvk: resid.Gvk{ + Group: "apps", + Version: "v1", + Kind: "Deployment", + }, + Name: "barman-cloud", + Namespace: "cnpg-system", + }, + }, + Options: nil, + }, + }, } - scheme := runtime.NewScheme() + scheme := cl.Scheme() if err := corev1.AddToScheme(scheme); err != nil { Fail(fmt.Sprintf("failed to add core/v1 to scheme: %v", err)) } @@ -93,6 +114,12 @@ var _ = SynchronizedBeforeSuite(func(ctx SpecContext) []byte { if err := certmanagerv1.AddToScheme(scheme); err != nil { Fail(fmt.Sprintf("failed to add cert-manager.io/v1 to scheme: %v", err)) } + if err := pluginBarmanCloudV1.AddToScheme(scheme); err != nil { + Fail(fmt.Sprintf("failed to add plugin-barman-cloud/v1 to scheme: %v", err)) + } + if err := cloudnativepgv1.AddToScheme(scheme); err != nil { + Fail(fmt.Sprintf("failed to add postgresql.cnpg.io/v1 to scheme: %v", err)) + } if err := kustomize.ApplyKustomization(ctx, cl, barmanCloudKustomization); err != nil { Fail(fmt.Sprintf("failed to apply kustomization: %v", err)) diff --git a/test/e2e/internal/certmanager/certmanager.go b/test/e2e/internal/certmanager/certmanager.go index f41aa36..0e1e2ed 100644 --- a/test/e2e/internal/certmanager/certmanager.go +++ b/test/e2e/internal/certmanager/certmanager.go @@ -29,23 +29,23 @@ import ( "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/kustomize" ) -// InstallOptions contains the options for installing cert-manager +// InstallOptions contains the options for installing cert-manager. type InstallOptions struct { Version string IgnoreExistResources bool } -// InstallOption is a function that sets up an option for installing cert-manager +// InstallOption is a function that sets up an option for installing cert-manager. type InstallOption func(*InstallOptions) -// WithVersion sets the version of cert-manager to install +// WithVersion sets the version of cert-manager to install. func WithVersion(version string) InstallOption { return func(opts *InstallOptions) { opts.Version = version } } -// WithIgnoreExistingResources sets whether to ignore existing resources +// WithIgnoreExistingResources sets whether to ignore existing resources. func WithIgnoreExistingResources(ignore bool) InstallOption { return func(opts *InstallOptions) { opts.IgnoreExistResources = ignore @@ -54,10 +54,10 @@ func WithIgnoreExistingResources(ignore bool) InstallOption { // TODO: renovate -// DefaultVersion is the default version of cert-manager to install +// DefaultVersion is the default version of cert-manager to install. const DefaultVersion = "v1.15.1" -// Install installs cert-manager using kubectl +// Install installs cert-manager using kubectl. func Install(ctx context.Context, cl client.Client, opts ...InstallOption) error { options := &InstallOptions{ Version: DefaultVersion, @@ -98,7 +98,7 @@ func Install(ctx context.Context, cl client.Client, opts ...InstallOption) error Namespace: "cert-manager", Name: deploymentName, }, interval); err != nil { - return err + return fmt.Errorf("failed to wait for deployment %s to be ready: %w", deploymentName, err) } } diff --git a/test/e2e/internal/client/client.go b/test/e2e/internal/client/client.go new file mode 100644 index 0000000..f444cde --- /dev/null +++ b/test/e2e/internal/client/client.go @@ -0,0 +1,56 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "fmt" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" +) + +// NewClient creates a new controller-runtime Kubernetes client. +// +//nolint:ireturn +func NewClient() (client.Client, *rest.Config, error) { + cfg, err := config.GetConfig() + if err != nil { + return nil, nil, fmt.Errorf("failed to get Kubernetes config: %w", err) + } + cl, err := client.New(cfg, client.Options{}) + if err != nil { + return nil, nil, fmt.Errorf("failed to create Kubernetes client: %w", err) + } + + return cl, cfg, nil +} + +// NewClientSet creates a new k8s client-go clientset. +func NewClientSet() (*kubernetes.Clientset, *rest.Config, error) { + cfg, err := config.GetConfig() + if err != nil { + return nil, nil, fmt.Errorf("failed to get Kubernetes config: %w", err) + } + clientSet, err := kubernetes.NewForConfig(cfg) + if err != nil { + return nil, nil, fmt.Errorf("failed to create Kubernetes client: %w", err) + } + + return clientSet, cfg, nil +} diff --git a/test/e2e/e2e_test.go b/test/e2e/internal/client/doc.go similarity index 69% rename from test/e2e/e2e_test.go rename to test/e2e/internal/client/doc.go index 570e673..7a28c4e 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/internal/client/doc.go @@ -14,17 +14,5 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -// const namespace = "plugin-barman-cloud-system" - -var _ = Describe("controller", Ordered, func() { - It("passes", func() { - Expect(true).To(BeTrue()) - }) -}) +// Package client provides function to create Kubernetes clients. +package client diff --git a/test/e2e/internal/cloudnativepg/cloudnativepg.go b/test/e2e/internal/cloudnativepg/cloudnativepg.go index d70b226..6325738 100644 --- a/test/e2e/internal/cloudnativepg/cloudnativepg.go +++ b/test/e2e/internal/cloudnativepg/cloudnativepg.go @@ -21,6 +21,7 @@ import ( "fmt" "time" + appsv1 "k8s.io/api/apps/v1" types2 "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/kustomize/api/types" @@ -30,7 +31,7 @@ import ( "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/kustomize" ) -// InstallCloudNativePGOptions contains the options for installing CloudNativePG +// InstallCloudNativePGOptions contains the options for installing CloudNativePG. type InstallCloudNativePGOptions struct { ImageName string ImageTag string @@ -40,52 +41,52 @@ type InstallCloudNativePGOptions struct { IgnoreExistResources bool } -// InstallOption is a function that sets up an option for installing CloudNativePG +// InstallOption is a function that sets up an option for installing CloudNativePG. type InstallOption func(*InstallCloudNativePGOptions) -// WithImageName sets the name for the CloudNativePG image +// WithImageName sets the name for the CloudNativePG image. func WithImageName(ref string) InstallOption { return func(opts *InstallCloudNativePGOptions) { opts.ImageName = ref } } -// WithImageTag sets the tag for the CloudNativePG image +// WithImageTag sets the tag for the CloudNativePG image. func WithImageTag(tag string) InstallOption { return func(opts *InstallCloudNativePGOptions) { opts.ImageTag = tag } } -// WithKustomizationResourceURL sets the URL for the CloudNativePG kustomization +// WithKustomizationResourceURL sets the URL for the CloudNativePG kustomization. func WithKustomizationResourceURL(url string) InstallOption { return func(opts *InstallCloudNativePGOptions) { opts.KustomizationResourceURL = url } } -// WithKustomizationRef sets the ref for the CloudNativePG kustomization +// WithKustomizationRef sets the ref for the CloudNativePG kustomization. func WithKustomizationRef(ref string) InstallOption { return func(opts *InstallCloudNativePGOptions) { opts.KustomizationRef = ref } } -// WithKustomizationTimeout sets the timeout for the kustomization resources +// WithKustomizationTimeout sets the timeout for the kustomization resources. func WithKustomizationTimeout(timeout string) InstallOption { return func(opts *InstallCloudNativePGOptions) { opts.KustomizationTimeout = timeout } } -// WithIgnoreExistingResources sets whether to ignore existing resources +// WithIgnoreExistingResources sets whether to ignore existing resources. func WithIgnoreExistingResources(ignore bool) InstallOption { return func(opts *InstallCloudNativePGOptions) { opts.IgnoreExistResources = ignore } } -// Install installs CloudNativePG using kubectl +// Install installs CloudNativePG using kubectl. func Install(ctx context.Context, cl client.Client, opts ...InstallOption) error { // Defining the default options options := &InstallCloudNativePGOptions{ @@ -129,6 +130,16 @@ func Install(ctx context.Context, cl client.Client, opts ...InstallOption) error }, } + // If the deployment exist, exit doing nothing + // If we redeploy, we'll zero out the webhook ca configuration and the tests will fail + if options.IgnoreExistResources { + deploy := &appsv1.Deployment{} + err := cl.Get(ctx, types2.NamespacedName{Namespace: "cnpg-system", Name: "cnpg-controller-manager"}, deploy) + if err == nil { + return nil + } + } + if err := kustomize.ApplyKustomization(ctx, cl, kustomization); err != nil { return fmt.Errorf("failed to apply kustomization: %w", err) } diff --git a/test/e2e/internal/cluster/cluster.go b/test/e2e/internal/cluster/cluster.go new file mode 100644 index 0000000..67d19bb --- /dev/null +++ b/test/e2e/internal/cluster/cluster.go @@ -0,0 +1,35 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cluster + +import ( + v1 "github.com/cloudnative-pg/api/pkg/api/v1" +) + +// TODO: improve this with what we already have in CloudNativePG e2e. +func IsReady(cluster v1.Cluster) bool { + if cluster.Status.ReadyInstances != cluster.Spec.Instances { + return false + } + for _, condition := range cluster.Status.Conditions { + if condition.Type == string(v1.ConditionClusterReady) { + return string(condition.Status) == string(v1.ConditionTrue) + } + } + + return false +} diff --git a/test/e2e/internal/cluster/doc.go b/test/e2e/internal/cluster/doc.go new file mode 100644 index 0000000..106c887 --- /dev/null +++ b/test/e2e/internal/cluster/doc.go @@ -0,0 +1,18 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package cluster contains functions to interact with the CloudNativePG clusters +package cluster diff --git a/test/e2e/internal/command/command.go b/test/e2e/internal/command/command.go new file mode 100644 index 0000000..7e841f2 --- /dev/null +++ b/test/e2e/internal/command/command.go @@ -0,0 +1,86 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package command + +import ( + "bytes" + "context" + "fmt" + "time" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" +) + +// TODO: extract this and the one in CloudNativePG to a common library + +// ContainerLocator is a struct that contains the information needed to locate a container in a pod. +type ContainerLocator struct { + NamespaceName string + PodName string + ContainerName string +} + +// ExecuteInContainer executes a command in a container. If timeout is not nil, the command will be +// executed with the specified timeout. The function returns the stdout and stderr of the command. +func ExecuteInContainer( + ctx context.Context, + clientSet kubernetes.Clientset, + cfg *rest.Config, + container ContainerLocator, + timeout *time.Duration, + command []string, +) (string, string, error) { + req := clientSet.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(container.PodName). + Namespace(container.NamespaceName). + SubResource("exec"). + Param("container", container.ContainerName). + Param("stdout", "true"). + Param("stderr", "true") + for _, cmd := range command { + req.Param("command", cmd) + } + + newConfig := *cfg // local copy avoids modifying the passed config arg + if timeout != nil { + req.Timeout(*timeout) + newConfig.Timeout = *timeout + timedCtx, cancelFunc := context.WithTimeout(ctx, *timeout) + defer cancelFunc() + ctx = timedCtx + } + + exec, err := remotecommand.NewSPDYExecutor(cfg, "POST", req.URL()) + if err != nil { + return "", "", fmt.Errorf("error creating executor: %w", err) + } + + var stdout, stderr bytes.Buffer + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + }) + if err != nil { + return "", "", fmt.Errorf("error executing command in pod '%s/%s': %w", + container.NamespaceName, container.PodName, err) + } + + return stdout.String(), stderr.String(), nil +} diff --git a/test/e2e/internal/command/doc.go b/test/e2e/internal/command/doc.go new file mode 100644 index 0000000..3691153 --- /dev/null +++ b/test/e2e/internal/command/doc.go @@ -0,0 +1,18 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package command provides function to execute commands in k8s pods. +package command diff --git a/test/e2e/internal/deployment/deployment.go b/test/e2e/internal/deployment/deployment.go index 6a40f08..b2ec73d 100644 --- a/test/e2e/internal/deployment/deployment.go +++ b/test/e2e/internal/deployment/deployment.go @@ -54,7 +54,7 @@ func IsReady(ctx context.Context, cl client.Client, name types.NamespacedName) ( func WaitForDeploymentReady( ctx context.Context, cl client.Client, namespacedName types.NamespacedName, interval time.Duration, ) error { - return wait.PollUntilContextCancel(ctx, interval, false, + err := wait.PollUntilContextCancel(ctx, interval, false, func(ctx context.Context) (bool, error) { ready, err := IsReady(ctx, cl, namespacedName) if err != nil { @@ -66,4 +66,9 @@ func WaitForDeploymentReady( return false, nil }) + if err != nil { + return fmt.Errorf("failed to wait for %s to be ready: %w", namespacedName, err) + } + + return nil } diff --git a/test/e2e/internal/e2etestenv/main.go b/test/e2e/internal/e2etestenv/main.go index 91b99df..089da81 100644 --- a/test/e2e/internal/e2etestenv/main.go +++ b/test/e2e/internal/e2etestenv/main.go @@ -23,10 +23,10 @@ import ( "time" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/config" "sigs.k8s.io/kind/pkg/cluster" "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/certmanager" + internalClient "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/client" "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/cloudnativepg" "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/kind" ) @@ -147,6 +147,8 @@ func defaultSetupOptions() SetupOptions { } // Setup sets up the test environment for the e2e tests, starting kind and installing the necessary components. +// +//nolint:ireturn func Setup(ctx context.Context, opts ...SetupOption) (client.Client, error) { options := defaultSetupOptions() for _, opt := range opts { @@ -157,9 +159,9 @@ func Setup(ctx context.Context, opts ...SetupOption) (client.Client, error) { return nil, err } - cl, err := getClient() + cl, _, err := internalClient.NewClient() if err != nil { - return nil, err + return nil, fmt.Errorf("failed to create Kubernetes client: %w", err) } if err := installCertManager(ctx, cl, options); err != nil { @@ -226,20 +228,6 @@ func installCertManager(ctx context.Context, cl client.Client, options SetupOpti return nil } -func getClient() (client.Client, error) { - // Use the current kubernetes client configuration - cfg, err := config.GetConfig() - if err != nil { - return nil, fmt.Errorf("failed to get Kubernetes config: %w", err) - } - cl, err := client.New(cfg, client.Options{}) - if err != nil { - return nil, fmt.Errorf("failed to create Kubernetes client: %w", err) - } - - return cl, nil -} - func setupKind(ctx context.Context, options SetupOptions) error { // This function sets up the environment for the e2e tests // by creating the cluster and installing the necessary diff --git a/test/e2e/internal/kind/cluster.go b/test/e2e/internal/kind/cluster.go index 6ee21ce..05c6803 100644 --- a/test/e2e/internal/kind/cluster.go +++ b/test/e2e/internal/kind/cluster.go @@ -19,7 +19,6 @@ package kind import ( "context" "fmt" - "os/exec" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/strslice" @@ -28,7 +27,7 @@ import ( "sigs.k8s.io/kind/pkg/cluster/nodes" ) -// IsClusterRunning checks if a Kind cluster with the given name is running +// IsClusterRunning checks if a Kind cluster with the given name is running. func IsClusterRunning(provider *cluster.Provider, clusterName string) (bool, error) { clusters, err := provider.List() if err != nil { @@ -43,38 +42,38 @@ func IsClusterRunning(provider *cluster.Provider, clusterName string) (bool, err return false, nil } -// CreateClusterOptions are the options for creating a Kind cluster +// CreateClusterOptions are the options for creating a Kind cluster. type CreateClusterOptions struct { ConfigFile string K8sVersion string Networks []string } -// CreateClusterOption is the option for creating a Kind cluster +// CreateClusterOption is the option for creating a Kind cluster. type CreateClusterOption func(*CreateClusterOptions) -// WithConfigFile sets the config file for creating a Kind cluster +// WithConfigFile sets the config file for creating a Kind cluster. func WithConfigFile(configFile string) CreateClusterOption { return func(opts *CreateClusterOptions) { opts.ConfigFile = configFile } } -// WithK8sVersion sets the Kubernetes version for creating a Kind cluster +// WithK8sVersion sets the Kubernetes version for creating a Kind cluster. func WithK8sVersion(k8sVersion string) CreateClusterOption { return func(opts *CreateClusterOptions) { opts.K8sVersion = k8sVersion } } -// WithNetwork sets the network for creating a Kind cluster +// WithNetworks sets the network for creating a Kind cluster. func WithNetworks(networks []string) CreateClusterOption { return func(opts *CreateClusterOptions) { opts.Networks = networks } } -// CreateCluster creates a Kind cluster with the given name +// CreateCluster creates a Kind cluster with the given name. func CreateCluster(ctx context.Context, provider *cluster.Provider, name string, opts ...CreateClusterOption) error { options := &CreateClusterOptions{} for _, opt := range opts { @@ -109,13 +108,20 @@ func CreateCluster(ctx context.Context, provider *cluster.Provider, name string, if err != nil { return err } - for _, node := range nodeList { - cmd := exec.Command("docker", "exec", node.String(), "update-ca-certificates") // #nosec - output, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("failed to update CA certificates in node %s: %w, output: %s", node, err, string(output)) - } + if err := updateCACertificates(ctx, cli, nodeList); err != nil { + return fmt.Errorf("failed to update CA certificates: %w", err) + } + + if err := connectNetworks(ctx, cli, nodeList, options.Networks); err != nil { + return fmt.Errorf("failed to connect networks: %w", err) + } + + return nil +} + +func updateCACertificates(ctx context.Context, cli *client.Client, nodes []nodes.Node) error { + for _, node := range nodes { execConfig := container.ExecOptions{ Cmd: strslice.StrSlice([]string{"update-ca-certificates"}), AttachStdout: true, @@ -132,8 +138,12 @@ func CreateCluster(ctx context.Context, provider *cluster.Provider, name string, } } - for _, netw := range options.Networks { - for _, node := range nodeList { + return nil +} + +func connectNetworks(ctx context.Context, cli *client.Client, nodes []nodes.Node, networks []string) error { + for _, netw := range networks { + for _, node := range nodes { err := cli.NetworkConnect(ctx, netw, node.String(), nil) if err != nil { return fmt.Errorf("failed to connect node %s to network %s: %w", node.String(), netw, err) diff --git a/test/e2e/internal/kustomize/kustomize.go b/test/e2e/internal/kustomize/kustomize.go index 4ea6d6f..414a5a5 100644 --- a/test/e2e/internal/kustomize/kustomize.go +++ b/test/e2e/internal/kustomize/kustomize.go @@ -22,7 +22,6 @@ import ( "errors" "fmt" "io" - "log" "gopkg.in/yaml.v3" apimachineryerrors "k8s.io/apimachinery/pkg/api/errors" @@ -34,15 +33,15 @@ import ( "sigs.k8s.io/kustomize/kyaml/filesys" ) -// ApplyKustomizationOptions holds options for applying kustomizations +// ApplyKustomizationOptions holds options for applying kustomizations. type ApplyKustomizationOptions struct { IgnoreExistingResources bool } -// ApplyKustomizationOption is a functional option for ApplyKustomization +// ApplyKustomizationOption is a functional option for ApplyKustomization. type ApplyKustomizationOption func(*ApplyKustomizationOptions) -// ApplyKustomization builds the kustomization and creates the resources +// ApplyKustomization builds the kustomization and creates the resources. func ApplyKustomization( ctx context.Context, cl client.Client, @@ -112,28 +111,32 @@ func applyResourceMap(ctx context.Context, cl client.Client, resourceMap resmap. } func applyResource(ctx context.Context, cl client.Client, obj *unstructured.Unstructured) error { - if err := cl.Create(ctx, obj); err != nil { - if apimachineryerrors.IsAlreadyExists(err) { - // If the resource already exists, retrieve the existing resource - existing := &unstructured.Unstructured{} - existing.SetGroupVersionKind(obj.GroupVersionKind()) - key := client.ObjectKey{ - Namespace: obj.GetNamespace(), - Name: obj.GetName(), - } - if err := cl.Get(ctx, key, existing); err != nil { - log.Fatalf("Error getting existing resource: %v", err) - } - - // Update the existing resource with the new data - obj.SetResourceVersion(existing.GetResourceVersion()) - err = cl.Update(ctx, obj) - if err != nil { - return fmt.Errorf("error updating resource: %v", err) - } - } else { - return fmt.Errorf("error creating resource: %v", err) - } + err := cl.Create(ctx, obj) + if err == nil { + return nil } + + if !apimachineryerrors.IsAlreadyExists(err) { + return fmt.Errorf("error creating resource: %w", err) + } + + // If the resource already exists, retrieve the existing resource + existing := &unstructured.Unstructured{} + existing.SetGroupVersionKind(obj.GroupVersionKind()) + key := client.ObjectKey{ + Namespace: obj.GetNamespace(), + Name: obj.GetName(), + } + if err := cl.Get(ctx, key, existing); err != nil { + return fmt.Errorf("error getting existing resource: %w", err) + } + + // Update the existing resource with the new data + obj.SetResourceVersion(existing.GetResourceVersion()) + err = cl.Update(ctx, obj) + if err != nil { + return fmt.Errorf("error updating resource: %w", err) + } + return nil } diff --git a/test/e2e/internal/namespace/doc.go b/test/e2e/internal/namespace/doc.go new file mode 100644 index 0000000..8c94cf9 --- /dev/null +++ b/test/e2e/internal/namespace/doc.go @@ -0,0 +1,18 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package namespace provides utilities to manage namespaces. +package namespace diff --git a/test/e2e/internal/namespace/namespace.go b/test/e2e/internal/namespace/namespace.go new file mode 100644 index 0000000..eb20845 --- /dev/null +++ b/test/e2e/internal/namespace/namespace.go @@ -0,0 +1,53 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package namespace + +import ( + "context" + "crypto/rand" + "fmt" + "math/big" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// CreateUniqueNamespace creates a namespace with an unique suffix. +func CreateUniqueNamespace(ctx context.Context, cl client.Client, prefix string) (*corev1.Namespace, error) { + for { + randInt, err := rand.Int(rand.Reader, big.NewInt(100000)) + if err != nil { + return nil, fmt.Errorf("failed to generate random number: %w", err) + } + namespaceName := fmt.Sprintf("%s-%d", prefix, randInt) + namespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespaceName, + }, + } + + err = cl.Create(ctx, namespace) + if err == nil { + return namespace, nil + } + if !apierrors.IsAlreadyExists(err) { + return nil, fmt.Errorf("failed to create namespace: %w", err) + } + } +} diff --git a/test/e2e/internal/objectstore/azurite.go b/test/e2e/internal/objectstore/azurite.go new file mode 100644 index 0000000..19ef603 --- /dev/null +++ b/test/e2e/internal/objectstore/azurite.go @@ -0,0 +1,203 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package objectstore + +import ( + "fmt" + + barmanapi "github.com/cloudnative-pg/barman-cloud/pkg/api" + "github.com/cloudnative-pg/machinery/pkg/api" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + + pluginBarmanCloudV1 "github.com/cloudnative-pg/plugin-barman-cloud/api/v1" +) + +func NewAzuriteObjectStoreResources(namespace, name string) Resources { + return Resources{ + Deployment: newAzuriteDeployment(namespace, name), + Service: newAzuriteService(namespace, name), + PVC: newAzuritePVC(namespace, name), + Secret: newAzuriteSecret(namespace, name), + } +} + +func newAzuriteDeployment(namespace, name string) *appsv1.Deployment { + return &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: ptr.To(int32(1)), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": name, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": name, + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: name, + // TODO: renovate the image + Image: "mcr.microsoft.com/azure-storage/azurite", + Ports: []corev1.ContainerPort{ + { + ContainerPort: 10000, + }, + }, + Env: []corev1.EnvVar{ + { + Name: "AZURITE_ACCOUNTS", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: name, + }, + Key: "AZURITE_ACCOUNTS", + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "data", + MountPath: "/data", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "data", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: name, + }, + }, + }, + }, + }, + }, + }, + } +} + +func newAzuriteService(namespace, name string) *corev1.Service { + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "app": name, + }, + Ports: []corev1.ServicePort{ + { + Port: 10000, + TargetPort: intstr.FromInt32(10000), + Protocol: corev1.ProtocolTCP, + }, + }, + }, + } +} + +func newAzuriteSecret(namespace, name string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + StringData: map[string]string{ + "AZURITE_ACCOUNTS": "storageaccountname:c3RvcmFnZWFjY291bnRrZXk=", + "AZURE_CONNECTION_STRING": "DefaultEndpointsProtocol=http;AccountName=storageaccountname;" + + "AccountKey=c3RvcmFnZWFjY291bnRrZXk=;" + + fmt.Sprintf("BlobEndpoint=http://%v:10000/storageaccountname;", name), + }, + } +} + +func newAzuritePVC(namespace, name string) *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ + TypeMeta: metav1.TypeMeta{ + Kind: "PersistentVolumeClaim", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse(DefaultSize), + }, + }, + }, + } +} + +func NewAzuriteObjectStore(namespace, name, azuriteOSName string) *pluginBarmanCloudV1.ObjectStore { + return &pluginBarmanCloudV1.ObjectStore{ + TypeMeta: metav1.TypeMeta{ + Kind: "ObjectStore", + APIVersion: "barmancloud.cnpg.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: pluginBarmanCloudV1.ObjectStoreSpec{ + Configuration: barmanapi.BarmanObjectStoreConfiguration{ + BarmanCredentials: barmanapi.BarmanCredentials{ + Azure: &barmanapi.AzureCredentials{ + ConnectionString: &api.SecretKeySelector{ + LocalObjectReference: api.LocalObjectReference{ + Name: azuriteOSName, + }, + Key: "AZURE_CONNECTION_STRING", + }, + }, + }, + DestinationPath: fmt.Sprintf("http://%v:10000/storageaccountname/backups/", azuriteOSName), + }, + }, + } +} diff --git a/test/e2e/internal/objectstore/doc.go b/test/e2e/internal/objectstore/doc.go new file mode 100644 index 0000000..021c8e8 --- /dev/null +++ b/test/e2e/internal/objectstore/doc.go @@ -0,0 +1,18 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package objectstore provides shared examples for object store resources. +package objectstore diff --git a/test/e2e/internal/objectstore/fakegcsserver.go b/test/e2e/internal/objectstore/fakegcsserver.go new file mode 100644 index 0000000..e2e6424 --- /dev/null +++ b/test/e2e/internal/objectstore/fakegcsserver.go @@ -0,0 +1,196 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package objectstore + +import ( + "fmt" + + barmanapi "github.com/cloudnative-pg/barman-cloud/pkg/api" + "github.com/cloudnative-pg/machinery/pkg/api" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + + pluginBarmanCloudV1 "github.com/cloudnative-pg/plugin-barman-cloud/api/v1" +) + +func NewGCSObjectStoreResources(namespace, name string) Resources { + return Resources{ + Deployment: newGCSDeployment(namespace, name), + Service: newGCSService(namespace, name), + Secret: newGCSSecret(namespace, name), + PVC: newGCSPVC(namespace, name), + } +} + +func newGCSDeployment(namespace, name string) *appsv1.Deployment { + return &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: ptr.To(int32(1)), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": name, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": name, + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: name, + // TODO: renovate the image + Image: "registry.barman-cloud-plugin:5000/fakegcs:test", + Ports: []corev1.ContainerPort{ + { + ContainerPort: 4443, + }, + }, + Command: []string{"fake-gcs-server"}, + Args: []string{"-scheme", + "http", + "-port", + "4443", + "-external-url", + fmt.Sprintf("http://%v:4443", name), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "storage", + MountPath: "/storage", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "storage", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: name, + }, + }, + }, + }, + }, + }, + }, + } +} + +func newGCSService(namespace, name string) *corev1.Service { + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "app": name, + }, + Ports: []corev1.ServicePort{ + { + Port: 4443, + TargetPort: intstr.FromInt32(4443), + Protocol: corev1.ProtocolTCP, + }, + }, + }, + } +} + +func newGCSSecret(namespace, name string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + StringData: map[string]string{ + "application_credentials": "", + }, + } +} + +func newGCSPVC(namespace, name string) *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ + TypeMeta: metav1.TypeMeta{ + Kind: "PersistentVolumeClaim", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse(DefaultSize), + }, + }, + }, + } +} + +func NewGCSObjectStore(namespace, name, gcsOSName string) *pluginBarmanCloudV1.ObjectStore { + return &pluginBarmanCloudV1.ObjectStore{ + TypeMeta: metav1.TypeMeta{ + Kind: "ObjectStore", + APIVersion: "barmancloud.cnpg.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: pluginBarmanCloudV1.ObjectStoreSpec{ + Configuration: barmanapi.BarmanObjectStoreConfiguration{ + BarmanCredentials: barmanapi.BarmanCredentials{ + Google: &barmanapi.GoogleCredentials{ + ApplicationCredentials: &api.SecretKeySelector{ + LocalObjectReference: api.LocalObjectReference{ + Name: gcsOSName, + }, + Key: "application_credentials", + }, + }, + }, + DestinationPath: "gs://backups/", + EndpointURL: fmt.Sprintf("http://%v:4443", gcsOSName), + }, + }, + } +} diff --git a/test/e2e/internal/objectstore/minio.go b/test/e2e/internal/objectstore/minio.go new file mode 100644 index 0000000..3f8ea48 --- /dev/null +++ b/test/e2e/internal/objectstore/minio.go @@ -0,0 +1,225 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package objectstore + +import ( + "net" + + barmanapi "github.com/cloudnative-pg/barman-cloud/pkg/api" + "github.com/cloudnative-pg/machinery/pkg/api" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + + pluginBarmanCloudV1 "github.com/cloudnative-pg/plugin-barman-cloud/api/v1" +) + +func NewMinioObjectStoreResources(namespace, name string) Resources { + return Resources{ + Deployment: newMinioDeployment(namespace, name), + Service: newMinioService(namespace, name), + PVC: newMinioPVC(namespace, name), + Secret: newMinioSecret(namespace, name), + } +} + +func newMinioDeployment(namespace, name string) *appsv1.Deployment { + return &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: ptr.To(int32(1)), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": name, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": name, + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: name, + // TODO: renovate the image + Image: "minio/minio:latest", + Args: []string{"server", "/data"}, + Ports: []corev1.ContainerPort{ + { + ContainerPort: 9000, + Name: name, + }, + }, + Env: []corev1.EnvVar{ + { + Name: "MINIO_ACCESS_KEY", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: name, + }, + Key: "ACCESS_KEY_ID", + }, + }, + }, + { + Name: "MINIO_SECRET_KEY", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: name, + }, + Key: "ACCESS_SECRET_KEY", + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "data", + MountPath: "/data", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "data", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: name, + }, + }, + }, + }, + }, + }, + }, + } +} + +func newMinioService(namespace, name string) *corev1.Service { + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "app": name, + }, + Ports: []corev1.ServicePort{ + { + Port: 9000, + TargetPort: intstr.FromInt32(9000), + Protocol: corev1.ProtocolTCP, + }, + }, + }, + } +} + +func newMinioSecret(namespace, name string) *corev1.Secret { + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Data: map[string][]byte{ + "ACCESS_KEY_ID": []byte("minio"), + "ACCESS_SECRET_KEY": []byte("minio123"), + }, + } +} + +func newMinioPVC(namespace, name string) *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ + TypeMeta: metav1.TypeMeta{ + Kind: "PersistentVolumeClaim", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse(DefaultSize), + }, + }, + }, + } +} + +func NewMinioObjectStore(namespace, name, minioOSName string) *pluginBarmanCloudV1.ObjectStore { + return &pluginBarmanCloudV1.ObjectStore{ + TypeMeta: metav1.TypeMeta{ + Kind: "ObjectStore", + APIVersion: "barmancloud.cnpg.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: pluginBarmanCloudV1.ObjectStoreSpec{ + Configuration: barmanapi.BarmanObjectStoreConfiguration{ + BarmanCredentials: barmanapi.BarmanCredentials{ + AWS: &barmanapi.S3Credentials{ + AccessKeyIDReference: &api.SecretKeySelector{ + LocalObjectReference: api.LocalObjectReference{ + Name: minioOSName, + }, + Key: "ACCESS_KEY_ID", + }, + SecretAccessKeyReference: &api.SecretKeySelector{ + LocalObjectReference: api.LocalObjectReference{ + Name: minioOSName, + }, + Key: "ACCESS_SECRET_KEY", + }, + }, + }, + EndpointURL: "http://" + net.JoinHostPort(minioOSName, "9000"), + DestinationPath: "s3://backups/", + }, + }, + } +} diff --git a/test/e2e/internal/objectstore/objectstore.go b/test/e2e/internal/objectstore/objectstore.go new file mode 100644 index 0000000..ca29e63 --- /dev/null +++ b/test/e2e/internal/objectstore/objectstore.go @@ -0,0 +1,57 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package objectstore + +import ( + "context" + "fmt" + + "k8s.io/api/apps/v1" + v2 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + // Size of the PVCs for the object stores + DefaultSize = "1Gi" +) + +// Resources represents the resources required to create an object store. +type Resources struct { + Deployment *v1.Deployment + Service *v2.Service + Secret *v2.Secret + PVC *v2.PersistentVolumeClaim +} + +// Create creates the object store resources. +func (osr Resources) Create(ctx context.Context, cl client.Client) error { + if err := cl.Create(ctx, osr.PVC); err != nil { + return fmt.Errorf("failed to create PVC: %w", err) + } + if err := cl.Create(ctx, osr.Secret); err != nil { + return fmt.Errorf("failed to create secret: %w", err) + } + if err := cl.Create(ctx, osr.Deployment); err != nil { + return fmt.Errorf("failed to create deployment: %w", err) + } + if err := cl.Create(ctx, osr.Service); err != nil { + return fmt.Errorf("failed to create service: %w", err) + } + + return nil +} diff --git a/test/e2e/internal/tests/backup/backup_restore.go b/test/e2e/internal/tests/backup/backup_restore.go new file mode 100644 index 0000000..a307b82 --- /dev/null +++ b/test/e2e/internal/tests/backup/backup_restore.go @@ -0,0 +1,211 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package backup + +import ( + "fmt" + "time" + + v1 "github.com/cloudnative-pg/api/pkg/api/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + internalClient "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/client" + cluster2 "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/cluster" + "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/command" + nmsp "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/namespace" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Backup and restore", func() { + var namespace *corev1.Namespace + var cl client.Client + BeforeEach(func(ctx SpecContext) { + var err error + cl, _, err = internalClient.NewClient() + Expect(err).NotTo(HaveOccurred()) + namespace, err = nmsp.CreateUniqueNamespace(ctx, cl, "backup-restore") + Expect(err).NotTo(HaveOccurred()) + }) + AfterEach(func(ctx SpecContext) { + Expect(cl.Delete(ctx, namespace)).To(Succeed()) + }) + + DescribeTable("should backup and restore a cluster", + func( + ctx SpecContext, + factory testCaseFactory, + ) { + testResources := factory.createBackupRestoreTestResources(namespace.Name) + + By("starting the ObjectStore deployment") + Expect(testResources.ObjectStoreResources.Create(ctx, cl)).To(Succeed()) + + By("creating the ObjectStore") + Expect(cl.Create(ctx, testResources.ObjectStore)).To(Succeed()) + + By("Creating a CloudNativePG cluster") + src := testResources.SrcCluster + Expect(cl.Create(ctx, testResources.SrcCluster)).To(Succeed()) + + By("Having the Cluster ready") + Eventually(func(g Gomega) { + g.Expect(cl.Get( + ctx, + types.NamespacedName{ + Name: src.Name, + Namespace: src.Namespace, + }, + src)).To(Succeed()) + g.Expect(cluster2.IsReady(*src)).To(BeTrue()) + }).WithTimeout(5 * time.Minute).WithPolling(5 * time.Second).Should(Succeed()) + + By("Adding data to PostgreSQL") + clientSet, cfg, err := internalClient.NewClientSet() + Expect(err).NotTo(HaveOccurred()) + _, _, err = command.ExecuteInContainer(ctx, + *clientSet, + cfg, + command.ContainerLocator{ + NamespaceName: src.Namespace, + PodName: fmt.Sprintf("%v-1", src.Name), + ContainerName: "postgres", + }, + nil, + []string{"psql", "-tAc", "CREATE TABLE test (i int); INSERT INTO test VALUES (1);"}) + Expect(err).NotTo(HaveOccurred()) + + By("Creating a backup") + backup := testResources.SrcBackup + Expect(cl.Create(ctx, backup)).To(Succeed()) + + By("Waiting for the backup to complete") + Eventually(func(g Gomega) { + g.Expect(cl.Get(ctx, types.NamespacedName{Name: backup.Name, Namespace: backup.Namespace}, + backup)).To(Succeed()) + g.Expect(backup.Status.Phase).To(BeEquivalentTo(v1.BackupPhaseCompleted)) + }).Within(2 * time.Minute).WithPolling(5 * time.Second).Should(Succeed()) + + By("Adding data after the backup") + _, _, err = command.ExecuteInContainer(ctx, + *clientSet, + cfg, + command.ContainerLocator{ + NamespaceName: src.Namespace, + PodName: fmt.Sprintf("%v-1", src.Name), + ContainerName: "postgres", + }, + nil, + []string{ + "psql", "-tAc", + "SELECT pg_switch_wal()" + + "; INSERT INTO test VALUES (2)", + }) + Expect(err).NotTo(HaveOccurred()) + _, _, err = command.ExecuteInContainer(ctx, + *clientSet, + cfg, + command.ContainerLocator{ + NamespaceName: src.Namespace, + PodName: fmt.Sprintf("%v-1", src.Name), + ContainerName: "postgres", + }, + nil, + []string{ + "psql", "-tAc", + "SELECT pg_switch_wal()", + }) + Expect(err).NotTo(HaveOccurred()) + + By("Restoring the backup") + dst := testResources.DstCluster + Expect(cl.Create(ctx, dst)).To(Succeed()) + + By("Having the Cluster ready") + Eventually(func(g Gomega) { + g.Expect(cl.Get(ctx, + types.NamespacedName{Name: dst.Name, Namespace: dst.Namespace}, + dst)).To(Succeed()) + g.Expect(cluster2.IsReady(*dst)).To(BeTrue()) + }).WithTimeout(5 * time.Minute).WithPolling(5 * time.Second).Should(Succeed()) + + By("Verifying the data exists in the restored instance") + output, _, err := command.ExecuteInContainer(ctx, + *clientSet, + cfg, + command.ContainerLocator{ + NamespaceName: dst.Namespace, + PodName: fmt.Sprintf("%v-1", dst.Name), + ContainerName: "postgres", + }, + nil, + []string{"psql", "-tAc", "SELECT count(*) FROM test;"}) + Expect(err).NotTo(HaveOccurred()) + Expect(output).To(BeEquivalentTo("2\n")) + + By("taking a backup from the restored cluster") + backup = testResources.DstBackup + Expect(cl.Create(ctx, backup)).To(Succeed()) + + By("Waiting for the backup to complete") + Eventually(func(g Gomega) { + g.Expect(cl.Get(ctx, types.NamespacedName{Name: backup.Name, Namespace: backup.Namespace}, + backup)).To(Succeed()) + g.Expect(backup.Status.Phase).To(BeEquivalentTo(v1.BackupPhaseCompleted)) + }).Within(2 * time.Minute).WithPolling(5 * time.Second).Should(Succeed()) + }, + Entry( + "using the plugin for backup and restore on S3", + &s3BackupPluginBackupPluginRestore{}, + ), + Entry( + "using the plugin for backup and in-tree for restore on S3", + &s3BackupPluginBackupInTreeRestore{}, + ), + Entry( + "using in-tree for backup and the plugin for restore on S3", + &s3BackupPluginInTreeBackupPluginRestore{}, + ), + Entry( + "using the plugin for backup and restore on Azure", + &azureBackupPluginBackupPluginRestore{}, + ), + Entry( + "using the plugin for backup and in-tree for restore on Azure", + &azureBackupPluginBackupInTreeRestore{}, + ), + Entry( + "using in-tree for backup and the plugin for restore on Azure", + &azureBackupPluginInTreeBackupPluginRestore{}, + ), + // TODO: enable the tests for GCS when we have support for STORAGE_EMULATOR_HOST + // env variable. + PEntry("using the plugin for backup and restore on GCS", + &gcsBackupPluginBackupPluginRestore{}, + ), + PEntry("using the plugin for backup and in-tree for restore on GCS", + &gcsBackupPluginBackupInTreeRestore{}, + ), + PEntry( + "using in-tree for backup and the plugin for restore on GCS", + &gcsBackupPluginInTreeBackupPluginRestore{}, + ), + ) +}) diff --git a/test/e2e/internal/tests/backup/doc.go b/test/e2e/internal/tests/backup/doc.go new file mode 100644 index 0000000..fe7051f --- /dev/null +++ b/test/e2e/internal/tests/backup/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package backup contains tests for the backup and restore functionality +// of the Barman Cloud Plugin. +package backup diff --git a/test/e2e/internal/tests/backup/fixtures.go b/test/e2e/internal/tests/backup/fixtures.go new file mode 100644 index 0000000..eb8a90a --- /dev/null +++ b/test/e2e/internal/tests/backup/fixtures.go @@ -0,0 +1,656 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package backup + +import ( + "fmt" + "net" + + cloudnativepgv1 "github.com/cloudnative-pg/api/pkg/api/v1" + barmanapi "github.com/cloudnative-pg/barman-cloud/pkg/api" + "github.com/cloudnative-pg/machinery/pkg/api" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + pluginBarmanCloudV1 "github.com/cloudnative-pg/plugin-barman-cloud/api/v1" + "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/objectstore" +) + +const ( + minio = "minio" + azurite = "azurite" + gcs = "gcs" + // Size of the PVCs for the object stores and the cluster instances + size = "1Gi" + srcClusterName = "source" + srcBackupName = "source" + objectStoreName = "source" + dstBackupName = "restore" + restoreClusterName = "restore" +) + +type testCaseFactory interface { + createBackupRestoreTestResources(namespace string) backupRestoreTestResources +} + +type backupRestoreTestResources struct { + ObjectStoreResources objectstore.Resources + ObjectStore *pluginBarmanCloudV1.ObjectStore + SrcCluster *cloudnativepgv1.Cluster + SrcBackup *cloudnativepgv1.Backup + DstCluster *cloudnativepgv1.Cluster + DstBackup *cloudnativepgv1.Backup +} + +type s3BackupPluginBackupPluginRestore struct{} + +func (s s3BackupPluginBackupPluginRestore) createBackupRestoreTestResources( + namespace string, +) backupRestoreTestResources { + result := backupRestoreTestResources{} + + result.ObjectStoreResources = objectstore.NewMinioObjectStoreResources(namespace, minio) + result.ObjectStore = objectstore.NewMinioObjectStore(namespace, objectStoreName, minio) + result.SrcCluster = newSrcClusterWithPlugin(namespace) + result.SrcBackup = newSrcPluginBackup(namespace) + result.DstCluster = newDstClusterWithPlugin(namespace) + result.DstBackup = newDstPluginBackup(namespace) + + return result +} + +type s3BackupPluginBackupInTreeRestore struct{} + +func (s s3BackupPluginBackupInTreeRestore) createBackupRestoreTestResources( + namespace string, +) backupRestoreTestResources { + result := backupRestoreTestResources{} + + result.ObjectStoreResources = objectstore.NewMinioObjectStoreResources(namespace, minio) + result.ObjectStore = objectstore.NewMinioObjectStore(namespace, objectStoreName, minio) + result.SrcCluster = newSrcClusterWithPlugin(namespace) + result.SrcBackup = newSrcPluginBackup(namespace) + result.DstCluster = newDstClusterInTreeS3(namespace) + result.DstBackup = newDstPluginBackup(namespace) + + return result +} + +type s3BackupPluginInTreeBackupPluginRestore struct{} + +func (s s3BackupPluginInTreeBackupPluginRestore) createBackupRestoreTestResources( + namespace string, +) backupRestoreTestResources { + result := backupRestoreTestResources{} + + result.ObjectStoreResources = objectstore.NewMinioObjectStoreResources(namespace, minio) + result.ObjectStore = objectstore.NewMinioObjectStore(namespace, objectStoreName, minio) + result.SrcCluster = newSrcClusterInTreeS3(namespace) + result.SrcBackup = newSrcInTreeBackup(namespace) + result.DstCluster = newDstClusterWithPlugin(namespace) + result.DstBackup = newDstPluginBackup(namespace) + + return result +} + +type azureBackupPluginBackupPluginRestore struct{} + +func (a azureBackupPluginBackupPluginRestore) createBackupRestoreTestResources( + namespace string, +) backupRestoreTestResources { + result := backupRestoreTestResources{} + + result.ObjectStoreResources = objectstore.NewAzuriteObjectStoreResources(namespace, azurite) + result.ObjectStore = objectstore.NewAzuriteObjectStore(namespace, objectStoreName, azurite) + result.SrcCluster = newSrcClusterWithPlugin(namespace) + result.SrcBackup = newSrcPluginBackup(namespace) + result.DstCluster = newDstClusterWithPlugin(namespace) + result.DstBackup = newDstPluginBackup(namespace) + + return result +} + +type azureBackupPluginBackupInTreeRestore struct{} + +func (a azureBackupPluginBackupInTreeRestore) createBackupRestoreTestResources( + namespace string, +) backupRestoreTestResources { + result := backupRestoreTestResources{} + + result.ObjectStoreResources = objectstore.NewAzuriteObjectStoreResources(namespace, azurite) + result.ObjectStore = objectstore.NewAzuriteObjectStore(namespace, objectStoreName, azurite) + result.SrcCluster = newSrcClusterWithPlugin(namespace) + result.SrcBackup = newSrcPluginBackup(namespace) + result.DstCluster = newDstClusterInTreeAzure(namespace) + result.DstBackup = newDstPluginBackup(namespace) + + return result +} + +type azureBackupPluginInTreeBackupPluginRestore struct{} + +func (a azureBackupPluginInTreeBackupPluginRestore) createBackupRestoreTestResources( + namespace string, +) backupRestoreTestResources { + result := backupRestoreTestResources{} + + result.ObjectStoreResources = objectstore.NewAzuriteObjectStoreResources(namespace, azurite) + result.ObjectStore = objectstore.NewAzuriteObjectStore(namespace, objectStoreName, azurite) + result.SrcCluster = newSrcClusterInTreeAzure(namespace) + result.SrcBackup = newSrcInTreeBackup(namespace) + result.DstCluster = newDstClusterWithPlugin(namespace) + result.DstBackup = newDstPluginBackup(namespace) + + return result +} + +type gcsBackupPluginBackupPluginRestore struct{} + +func (g gcsBackupPluginBackupPluginRestore) createBackupRestoreTestResources( + namespace string, +) backupRestoreTestResources { + result := backupRestoreTestResources{} + + result.ObjectStoreResources = objectstore.NewGCSObjectStoreResources(namespace, gcs) + result.ObjectStore = objectstore.NewGCSObjectStore(namespace, objectStoreName, gcs) + result.SrcCluster = newSrcClusterWithPlugin(namespace) + result.SrcBackup = newSrcPluginBackup(namespace) + result.DstCluster = newDstClusterWithPlugin(namespace) + result.DstBackup = newDstPluginBackup(namespace) + + return result +} + +type gcsBackupPluginBackupInTreeRestore struct{} + +func (g gcsBackupPluginBackupInTreeRestore) createBackupRestoreTestResources( + namespace string, +) backupRestoreTestResources { + result := backupRestoreTestResources{} + + result.ObjectStoreResources = objectstore.NewGCSObjectStoreResources(namespace, gcs) + result.ObjectStore = objectstore.NewGCSObjectStore(namespace, objectStoreName, gcs) + result.SrcCluster = newSrcClusterWithPlugin(namespace) + result.SrcBackup = newSrcPluginBackup(namespace) + result.DstCluster = newDstClusterInTreeGCS(namespace) + result.DstBackup = newDstPluginBackup(namespace) + + return result +} + +type gcsBackupPluginInTreeBackupPluginRestore struct{} + +func (g gcsBackupPluginInTreeBackupPluginRestore) createBackupRestoreTestResources( + namespace string, +) backupRestoreTestResources { + result := backupRestoreTestResources{} + + result.ObjectStoreResources = objectstore.NewGCSObjectStoreResources(namespace, gcs) + result.ObjectStore = objectstore.NewGCSObjectStore(namespace, objectStoreName, gcs) + result.SrcCluster = newSrcClusterInTreeGCS(namespace) + result.SrcBackup = newSrcInTreeBackup(namespace) + result.DstCluster = newDstClusterWithPlugin(namespace) + result.DstBackup = newDstPluginBackup(namespace) + + return result +} + +func newSrcPluginBackup(namespace string) *cloudnativepgv1.Backup { + return &cloudnativepgv1.Backup{ + TypeMeta: metav1.TypeMeta{ + Kind: "Backup", + APIVersion: "postgresql.cnpg.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: srcBackupName, + Namespace: namespace, + }, + Spec: cloudnativepgv1.BackupSpec{ + Cluster: cloudnativepgv1.LocalObjectReference{ + Name: srcClusterName, + }, + Method: "plugin", + PluginConfiguration: &cloudnativepgv1.BackupPluginConfiguration{ + Name: "barman-cloud.cloudnative-pg.io", + }, + }, + } +} + +func newSrcInTreeBackup(namespace string) *cloudnativepgv1.Backup { + return &cloudnativepgv1.Backup{ + TypeMeta: metav1.TypeMeta{ + Kind: "Backup", + APIVersion: "postgresql.cnpg.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: srcBackupName, + Namespace: namespace, + }, + Spec: cloudnativepgv1.BackupSpec{ + Cluster: cloudnativepgv1.LocalObjectReference{ + Name: srcClusterName, + }, + Method: "barmanObjectStore", + }, + } +} + +func newDstPluginBackup(namespace string) *cloudnativepgv1.Backup { + return &cloudnativepgv1.Backup{ + TypeMeta: metav1.TypeMeta{ + Kind: "Backup", + APIVersion: "postgresql.cnpg.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: dstBackupName, + Namespace: namespace, + }, + Spec: cloudnativepgv1.BackupSpec{ + Cluster: cloudnativepgv1.LocalObjectReference{ + Name: restoreClusterName, + }, + Method: "plugin", + PluginConfiguration: &cloudnativepgv1.BackupPluginConfiguration{ + Name: "barman-cloud.cloudnative-pg.io", + }, + }, + } +} + +func newSrcClusterWithPlugin(namespace string) *cloudnativepgv1.Cluster { + cluster := &cloudnativepgv1.Cluster{ + TypeMeta: metav1.TypeMeta{ + Kind: "Cluster", + APIVersion: "postgresql.cnpg.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: srcClusterName, + Namespace: namespace, + }, + Spec: cloudnativepgv1.ClusterSpec{ + Instances: 2, + ImagePullPolicy: corev1.PullAlways, + Plugins: cloudnativepgv1.PluginConfigurationList{ + { + Name: "barman-cloud.cloudnative-pg.io", + Parameters: map[string]string{ + "barmanObjectName": objectStoreName, + }, + }, + }, + PostgresConfiguration: cloudnativepgv1.PostgresConfiguration{ + Parameters: map[string]string{ + "log_min_messages": "DEBUG4", + }, + }, + StorageConfiguration: cloudnativepgv1.StorageConfiguration{ + Size: size, + }, + }, + } + + return cluster +} + +func newDstClusterWithPlugin(namespace string) *cloudnativepgv1.Cluster { + cluster := &cloudnativepgv1.Cluster{ + TypeMeta: metav1.TypeMeta{ + Kind: "Cluster", + APIVersion: "postgresql.cnpg.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: restoreClusterName, + Namespace: namespace, + }, + Spec: cloudnativepgv1.ClusterSpec{ + Instances: 2, + ImagePullPolicy: corev1.PullAlways, + Bootstrap: &cloudnativepgv1.BootstrapConfiguration{ + Recovery: &cloudnativepgv1.BootstrapRecovery{ + Source: "source", + }, + }, + Plugins: cloudnativepgv1.PluginConfigurationList{ + { + Name: "barman-cloud.cloudnative-pg.io", + Parameters: map[string]string{ + "barmanObjectName": objectStoreName, + }, + }, + }, + PostgresConfiguration: cloudnativepgv1.PostgresConfiguration{ + Parameters: map[string]string{ + "log_min_messages": "DEBUG4", + }, + }, + ExternalClusters: []cloudnativepgv1.ExternalCluster{ + { + Name: "source", + PluginConfiguration: &cloudnativepgv1.PluginConfiguration{ + Name: "barman-cloud.cloudnative-pg.io", + Parameters: map[string]string{ + "barmanObjectName": objectStoreName, + "serverName": srcClusterName, + }, + }, + }, + }, + StorageConfiguration: cloudnativepgv1.StorageConfiguration{ + Size: size, + }, + }, + } + + return cluster +} + +func newSrcClusterInTreeS3(namespace string) *cloudnativepgv1.Cluster { + cluster := &cloudnativepgv1.Cluster{ + TypeMeta: metav1.TypeMeta{ + Kind: "Cluster", + APIVersion: "postgresql.cnpg.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: srcClusterName, + Namespace: namespace, + }, + Spec: cloudnativepgv1.ClusterSpec{ + Instances: 2, + ImagePullPolicy: corev1.PullAlways, + StorageConfiguration: cloudnativepgv1.StorageConfiguration{ + Size: size, + }, + PostgresConfiguration: cloudnativepgv1.PostgresConfiguration{ + Parameters: map[string]string{ + "log_min_messages": "DEBUG4", + }, + }, + Backup: &cloudnativepgv1.BackupConfiguration{ + BarmanObjectStore: &cloudnativepgv1.BarmanObjectStoreConfiguration{ + BarmanCredentials: barmanapi.BarmanCredentials{ + AWS: &barmanapi.S3Credentials{ + AccessKeyIDReference: &api.SecretKeySelector{ + LocalObjectReference: api.LocalObjectReference{ + Name: minio, + }, + Key: "ACCESS_KEY_ID", + }, + SecretAccessKeyReference: &api.SecretKeySelector{ + LocalObjectReference: api.LocalObjectReference{ + Name: minio, + }, + Key: "ACCESS_SECRET_KEY", + }, + }, + }, + EndpointURL: "http://" + net.JoinHostPort(minio, "9000"), + DestinationPath: "s3://backups/", + }, + }, + }, + } + + return cluster +} + +func newDstClusterInTreeS3(namespace string) *cloudnativepgv1.Cluster { + cluster := &cloudnativepgv1.Cluster{ + TypeMeta: metav1.TypeMeta{ + Kind: "Cluster", + APIVersion: "postgresql.cnpg.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: restoreClusterName, + Namespace: namespace, + }, + Spec: cloudnativepgv1.ClusterSpec{ + Instances: 2, + ImagePullPolicy: corev1.PullAlways, + Bootstrap: &cloudnativepgv1.BootstrapConfiguration{ + Recovery: &cloudnativepgv1.BootstrapRecovery{ + Source: "source", + }, + }, + PostgresConfiguration: cloudnativepgv1.PostgresConfiguration{ + Parameters: map[string]string{ + "log_min_messages": "DEBUG4", + }, + }, + Plugins: cloudnativepgv1.PluginConfigurationList{ + { + Name: "barman-cloud.cloudnative-pg.io", + Parameters: map[string]string{ + "barmanObjectName": objectStoreName, + }, + }, + }, + ExternalClusters: []cloudnativepgv1.ExternalCluster{ + { + Name: "source", + BarmanObjectStore: &cloudnativepgv1.BarmanObjectStoreConfiguration{ + BarmanCredentials: barmanapi.BarmanCredentials{ + AWS: &barmanapi.S3Credentials{ + AccessKeyIDReference: &api.SecretKeySelector{ + LocalObjectReference: api.LocalObjectReference{ + Name: minio, + }, + Key: "ACCESS_KEY_ID", + }, + SecretAccessKeyReference: &api.SecretKeySelector{ + LocalObjectReference: api.LocalObjectReference{ + Name: minio, + }, + Key: "ACCESS_SECRET_KEY", + }, + }, + }, + EndpointURL: "http://" + net.JoinHostPort(minio, "9000"), + DestinationPath: "s3://backups/", + }, + }, + }, + StorageConfiguration: cloudnativepgv1.StorageConfiguration{ + Size: size, + }, + }, + } + + return cluster +} + +func newSrcClusterInTreeAzure(namespace string) *cloudnativepgv1.Cluster { + cluster := &cloudnativepgv1.Cluster{ + TypeMeta: metav1.TypeMeta{ + Kind: "Cluster", + APIVersion: "postgresql.cnpg.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: srcClusterName, + Namespace: namespace, + }, + Spec: cloudnativepgv1.ClusterSpec{ + Instances: 2, + ImagePullPolicy: corev1.PullAlways, + StorageConfiguration: cloudnativepgv1.StorageConfiguration{ + Size: size, + }, + Backup: &cloudnativepgv1.BackupConfiguration{ + BarmanObjectStore: &cloudnativepgv1.BarmanObjectStoreConfiguration{ + BarmanCredentials: barmanapi.BarmanCredentials{ + Azure: &barmanapi.AzureCredentials{ + ConnectionString: &api.SecretKeySelector{ + LocalObjectReference: api.LocalObjectReference{ + Name: azurite, + }, + Key: "AZURE_CONNECTION_STRING", + }, + }, + }, + DestinationPath: fmt.Sprintf("http://%v:10000/storageaccountname/backups/", azurite), + }, + }, + }, + } + + return cluster +} + +func newDstClusterInTreeAzure(namespace string) *cloudnativepgv1.Cluster { + cluster := &cloudnativepgv1.Cluster{ + TypeMeta: metav1.TypeMeta{ + Kind: "Cluster", + APIVersion: "postgresql.cnpg.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: restoreClusterName, + Namespace: namespace, + }, + Spec: cloudnativepgv1.ClusterSpec{ + Instances: 2, + ImagePullPolicy: corev1.PullAlways, + Bootstrap: &cloudnativepgv1.BootstrapConfiguration{ + Recovery: &cloudnativepgv1.BootstrapRecovery{ + Source: "source", + }, + }, + Plugins: cloudnativepgv1.PluginConfigurationList{ + { + Name: "barman-cloud.cloudnative-pg.io", + Parameters: map[string]string{ + "barmanObjectName": objectStoreName, + }, + }, + }, + ExternalClusters: []cloudnativepgv1.ExternalCluster{ + { + Name: "source", + BarmanObjectStore: &cloudnativepgv1.BarmanObjectStoreConfiguration{ + BarmanCredentials: barmanapi.BarmanCredentials{ + Azure: &barmanapi.AzureCredentials{ + ConnectionString: &api.SecretKeySelector{ + LocalObjectReference: api.LocalObjectReference{ + Name: azurite, + }, + Key: "AZURE_CONNECTION_STRING", + }, + }, + }, + DestinationPath: fmt.Sprintf("http://%v:10000/storageaccountname/backups/", azurite), + }, + }, + }, + StorageConfiguration: cloudnativepgv1.StorageConfiguration{ + Size: size, + }, + }, + } + + return cluster +} + +func newSrcClusterInTreeGCS(namespace string) *cloudnativepgv1.Cluster { + cluster := &cloudnativepgv1.Cluster{ + TypeMeta: metav1.TypeMeta{ + Kind: "Cluster", + APIVersion: "postgresql.cnpg.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: srcClusterName, + Namespace: namespace, + }, + Spec: cloudnativepgv1.ClusterSpec{ + Instances: 2, + ImagePullPolicy: corev1.PullAlways, + StorageConfiguration: cloudnativepgv1.StorageConfiguration{ + Size: size, + }, + Backup: &cloudnativepgv1.BackupConfiguration{ + BarmanObjectStore: &cloudnativepgv1.BarmanObjectStoreConfiguration{ + BarmanCredentials: barmanapi.BarmanCredentials{ + Google: &barmanapi.GoogleCredentials{ + ApplicationCredentials: &api.SecretKeySelector{ + LocalObjectReference: api.LocalObjectReference{ + Name: gcs, + }, + Key: "application_credentials", + }, + }, + }, + EndpointURL: fmt.Sprintf("http://%v:4443", gcs), + DestinationPath: "gs://backups/", + }, + }, + }, + } + + return cluster +} + +func newDstClusterInTreeGCS(namespace string) *cloudnativepgv1.Cluster { + cluster := &cloudnativepgv1.Cluster{ + TypeMeta: metav1.TypeMeta{ + Kind: "Cluster", + APIVersion: "postgresql.cnpg.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: restoreClusterName, + Namespace: namespace, + }, + Spec: cloudnativepgv1.ClusterSpec{ + Instances: 2, + ImagePullPolicy: corev1.PullAlways, + Bootstrap: &cloudnativepgv1.BootstrapConfiguration{ + Recovery: &cloudnativepgv1.BootstrapRecovery{ + Source: "source", + }, + }, + Plugins: cloudnativepgv1.PluginConfigurationList{ + { + Name: "barman-cloud.cloudnative-pg.io", + Parameters: map[string]string{ + "barmanObjectName": objectStoreName, + }, + }, + }, + ExternalClusters: []cloudnativepgv1.ExternalCluster{ + { + Name: "source", + BarmanObjectStore: &cloudnativepgv1.BarmanObjectStoreConfiguration{ + BarmanCredentials: barmanapi.BarmanCredentials{ + Google: &barmanapi.GoogleCredentials{ + ApplicationCredentials: &api.SecretKeySelector{ + LocalObjectReference: api.LocalObjectReference{ + Name: gcs, + }, + Key: "application_credentials", + }, + }, + }, + DestinationPath: "gs://backups/", + EndpointURL: fmt.Sprintf("http://%v:4443", gcs), + }, + }, + }, + StorageConfiguration: cloudnativepgv1.StorageConfiguration{ + Size: size, + }, + }, + } + + return cluster +}