feat(spike): backup method (#20)

Signed-off-by: Armando Ruocco <armando.ruocco@enterprisedb.com>
Signed-off-by: Leonardo Cecchi <leonardo.cecchi@enterprisedb.com>
Signed-off-by: Francesco Canovai <francesco.canovai@enterprisedb.com>
Co-authored-by: Leonardo Cecchi <leonardo.cecchi@enterprisedb.com>
Co-authored-by: Francesco Canovai <francesco.canovai@enterprisedb.com>

docs/examples/backup-example.yaml

@@ -0,0 +1,12 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Backup
+metadata:
+  name: backup-example
+spec:
+  method: plugin
+
+  cluster:
+    name: cluster-example
+
+  pluginConfiguration:
+    name: barman-cloud.cloudnative-pg.io

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/cloudnative-pg/cloudnative-pg v1.24.1-0.20241001084914-829808376542
 	github.com/cloudnative-pg/cnpg-i v0.0.0-20240924030516-c5636170f248
 	github.com/cloudnative-pg/cnpg-i-machinery v0.0.0-20241002070940-e5495e9c5ed6
-	github.com/cloudnative-pg/machinery v0.0.0-20241001075747-34c8797af80f
+	github.com/cloudnative-pg/machinery v0.0.0-20241007093555-1e197af1f392
 	github.com/onsi/ginkgo/v2 v2.20.2
 	github.com/onsi/gomega v1.34.2
 	github.com/spf13/cobra v1.8.1

go.sum

@@ -22,8 +22,8 @@ github.com/cloudnative-pg/cnpg-i v0.0.0-20240924030516-c5636170f248 h1:eUGzb7YNj
 github.com/cloudnative-pg/cnpg-i v0.0.0-20240924030516-c5636170f248/go.mod h1:K9/4eAT3rh2bKIWyujoN8BIPRXa4d1Ls+eBY8PE8y6w=
 github.com/cloudnative-pg/cnpg-i-machinery v0.0.0-20241002070940-e5495e9c5ed6 h1:C4CU5fBTYTiJBPDqcgHpXSc5IvRTy+5KTaFZzdKHfAQ=
 github.com/cloudnative-pg/cnpg-i-machinery v0.0.0-20241002070940-e5495e9c5ed6/go.mod h1:mHEVy/Guae+rij1qlgwHg+lyFKDX48qjTL4lAqE7OJs=
-github.com/cloudnative-pg/machinery v0.0.0-20241001075747-34c8797af80f h1:RgPmQJkuSu3eTdfd4T2K95RYQi57LHB2+Jfsu/faKOM=
+github.com/cloudnative-pg/machinery v0.0.0-20241007093555-1e197af1f392 h1:DHaSe0PoLnIQFWIpRqB9RiBlNzbdLuVbiCtc9tN+FL0=
-github.com/cloudnative-pg/machinery v0.0.0-20241001075747-34c8797af80f/go.mod h1:bWp1Es5zlxElg4Z/c5f0RKOkDcyNvDHdYIvNcPQU4WM=
+github.com/cloudnative-pg/machinery v0.0.0-20241007093555-1e197af1f392/go.mod h1:bWp1Es5zlxElg4Z/c5f0RKOkDcyNvDHdYIvNcPQU4WM=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

internal/cnpgi/instance/backup.go

@@ -2,16 +2,46 @@ package instance
 
 import (
 	"context"
+	"fmt"
+	"os"
+	"strconv"
+	"time"
 
+	barmanBackup "github.com/cloudnative-pg/barman-cloud/pkg/backup"
+	barmanCapabilities "github.com/cloudnative-pg/barman-cloud/pkg/capabilities"
+	barmanCredentials "github.com/cloudnative-pg/barman-cloud/pkg/credentials"
+	"github.com/cloudnative-pg/cloudnative-pg/pkg/postgres"
 	"github.com/cloudnative-pg/cnpg-i/pkg/backup"
+	"github.com/cloudnative-pg/machinery/pkg/fileutils"
+	"github.com/cloudnative-pg/machinery/pkg/log"
+	pgTime "github.com/cloudnative-pg/machinery/pkg/postgres/time"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	barmancloudv1 "github.com/cloudnative-pg/plugin-barman-cloud/api/v1"
+	"github.com/cloudnative-pg/plugin-barman-cloud/internal/cnpgi/metadata"
 )
 
 // BackupServiceImplementation is the implementation
 // of the Backup CNPG capability
 type BackupServiceImplementation struct {
+	BarmanObjectKey  client.ObjectKey
+	ClusterObjectKey client.ObjectKey
+	Client           client.Client
+	InstanceName     string
 	backup.UnimplementedBackupServer
 }
 
+// This is an implementation of the barman executor
+// that always instruct the barman library to use the
+// "--name" option for backups. We don't support old
+// Barman versions that do not implement that option.
+type barmanCloudExecutor struct{}
+
+func (barmanCloudExecutor) ShouldForceLegacyBackup() bool {
+	return false
+}
+
 // GetCapabilities implements the BackupService interface
 func (b BackupServiceImplementation) GetCapabilities(
 	_ context.Context, _ *backup.BackupCapabilitiesRequest,
@@ -30,7 +60,86 @@ func (b BackupServiceImplementation) GetCapabilities(
 }
 
 // Backup implements the Backup interface
-func (b BackupServiceImplementation) Backup(_ context.Context, _ *backup.BackupRequest) (*backup.BackupResult, error) {
+func (b BackupServiceImplementation) Backup(
-	// TODO implement me
+	ctx context.Context,
-	panic("implement me")
+	_ *backup.BackupRequest,
+) (*backup.BackupResult, error) {
+	contextLogger := log.FromContext(ctx)
+
+	var objectStore barmancloudv1.ObjectStore
+	if err := b.Client.Get(ctx, b.BarmanObjectKey, &objectStore); err != nil {
+		return nil, err
+	}
+
+	if err := fileutils.EnsureDirectoryExists(postgres.BackupTemporaryDirectory); err != nil {
+		contextLogger.Error(err, "Cannot create backup temporary directory", "err", err)
+		return nil, err
+	}
+
+	capabilities, err := barmanCapabilities.CurrentCapabilities()
+	if err != nil {
+		return nil, err
+	}
+	backupCmd := barmanBackup.NewBackupCommand(
+		&objectStore.Spec.Configuration,
+		capabilities,
+	)
+
+	// We need to connect to PostgreSQL and to do that we need
+	// PGHOST (and the like) to be available
+	osEnvironment := os.Environ()
+	caBundleEnvironment := getRestoreCABundleEnv(&objectStore.Spec.Configuration)
+	env, err := barmanCredentials.EnvSetBackupCloudCredentials(
+		ctx,
+		b.Client,
+		objectStore.Namespace,
+		&objectStore.Spec.Configuration,
+		mergeEnv(osEnvironment, caBundleEnvironment))
+	if err != nil {
+		return nil, err
+	}
+
+	backupName := fmt.Sprintf("backup-%v", pgTime.ToCompactISO8601(time.Now()))
+
+	if err = backupCmd.Take(
+		ctx,
+		backupName,
+		b.InstanceName,
+		env,
+		barmanCloudExecutor{},
+		postgres.BackupTemporaryDirectory,
+	); err != nil {
+		return nil, err
+	}
+
+	executedBackupInfo, err := backupCmd.GetExecutedBackupInfo(
+		ctx,
+		backupName,
+		b.InstanceName,
+		barmanCloudExecutor{},
+		env)
+	if err != nil {
+		return nil, err
+	}
+
+	return &backup.BackupResult{
+		BackupId:          executedBackupInfo.ID,
+		BackupName:        executedBackupInfo.BackupName,
+		StartedAt:         metav1.Time{Time: executedBackupInfo.BeginTime}.Unix(),
+		StoppedAt:         metav1.Time{Time: executedBackupInfo.EndTime}.Unix(),
+		BeginWal:          executedBackupInfo.BeginWal,
+		EndWal:            executedBackupInfo.EndWal,
+		BeginLsn:          executedBackupInfo.BeginLSN,
+		EndLsn:            executedBackupInfo.EndLSN,
+		BackupLabelFile:   nil,
+		TablespaceMapFile: nil,
+		InstanceId:        b.InstanceName,
+		Online:            true,
+		Metadata: map[string]string{
+			"timeline":    strconv.Itoa(executedBackupInfo.TimeLine),
+			"version":     metadata.Data.Version,
+			"name":        metadata.Data.Name,
+			"displayName": metadata.Data.DisplayName,
+		},
+	}, nil
 }

internal/cnpgi/instance/start.go

@@ -35,7 +35,12 @@ func (c *CNPGI) Start(ctx context.Context) error {
 			PGDataPath:       c.PGDataPath,
 			PGWALPath:        c.PGWALPath,
 		})
-		backup.RegisterBackupServer(server, BackupServiceImplementation{})
+		backup.RegisterBackupServer(server, BackupServiceImplementation{
+			Client:           c.Client,
+			BarmanObjectKey:  c.BarmanObjectKey,
+			ClusterObjectKey: c.ClusterObjectKey,
+			InstanceName:     c.InstanceName,
+		})
 		return nil
 	}
 

internal/cnpgi/instance/wal.go

@@ -124,9 +124,8 @@ func (w WALServiceImplementation) Restore(
 	contextLogger := log.FromContext(ctx)
 	startTime := time.Now()
 
-	var cluster *cnpgv1.Cluster
+	var cluster cnpgv1.Cluster
-
+	if err := w.Client.Get(ctx, w.ClusterObjectKey, &cluster); err != nil {
-	if err := w.Client.Get(ctx, w.ClusterObjectKey, cluster); err != nil {
 		return nil, err
 	}
 
@@ -152,7 +151,7 @@ func (w WALServiceImplementation) Restore(
 	if err != nil {
 		return nil, fmt.Errorf("while getting recover credentials: %w", err)
 	}
-	mergeEnv(env, credentialsEnv)
+	env = mergeEnv(env, credentialsEnv)
 
 	options, err := barmanCommand.CloudWalRestoreOptions(ctx, barmanConfiguration, objectStore.Name)
 	if err != nil {
@@ -178,7 +177,7 @@ func (w WALServiceImplementation) Restore(
 	}
 
 	// We skip this step if streaming connection is not available
-	if isStreamingAvailable(cluster, w.InstanceName) {
+	if isStreamingAvailable(&cluster, w.InstanceName) {
 		if err := checkEndOfWALStreamFlag(walRestorer); err != nil {
 			return nil, err
 		}
@@ -213,7 +212,7 @@ func (w WALServiceImplementation) Restore(
 
 	// We skip this step if streaming connection is not available
 	endOfWALStream := isEndOfWALStream(walStatus)
-	if isStreamingAvailable(cluster, w.InstanceName) && endOfWALStream {
+	if isStreamingAvailable(&cluster, w.InstanceName) && endOfWALStream {
 		contextLogger.Info(
 			"Set end-of-wal-stream flag as one of the WAL files to be prefetched was not found")
 
@@ -262,18 +261,29 @@ func (w WALServiceImplementation) SetFirstRequired(
 }
 
 // mergeEnv merges all the values inside incomingEnv into env.
-func mergeEnv(env []string, incomingEnv []string) {
+func mergeEnv(env []string, incomingEnv []string) []string {
+	result := make([]string, len(env), len(env)+len(incomingEnv))
+	copy(result, env)
+
 	for _, incomingItem := range incomingEnv {
 		incomingKV := strings.SplitAfterN(incomingItem, "=", 2)
 		if len(incomingKV) != 2 {
 			continue
 		}
-		for idx, item := range env {
+
+		found := false
+		for idx, item := range result {
 			if strings.HasPrefix(item, incomingKV[0]) {
-				env[idx] = incomingItem
+				result[idx] = incomingItem
+				found = true
 			}
 		}
+		if !found {
+			result = append(result, incomingItem)
+		}
 	}
+
+	return result
 }
 
 // TODO: refactor.

scripts/cleanup.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -eu
+
+cd "$(dirname "$0")/.." || exit
+
+kubectl delete clusters --all
+kubectl delete backups --all
+kubectl exec -ti mc -- mc rm -r --force minio/backups

scripts/run.sh

@@ -8,15 +8,19 @@ if [ -f .env ]; then
     source .env
 fi
 
+
+MYTMPDIR="$(mktemp -d)"
+trap 'rm -rf -- "$MYTMPDIR"' EXIT
+
 current_context=$(kubectl config view --raw -o json | jq -r '."current-context"' | sed "s/kind-//")
 operator_image=$(KIND_CLUSTER_NAME="$current_context" KO_DOCKER_REPO=kind.local ko build -BP ./cmd/operator)
 instance_image=$(KIND_CLUSTER_NAME="$current_context" KO_DOCKER_REPO=kind.local KO_DEFAULTBASEIMAGE="ghcr.io/cloudnative-pg/postgresql:17.0" ko build -BP ./cmd/instance)
 
+# Now we deploy the plugin inside the `cnpg-system` workspace
 (
-  cd kubernetes;
+  cp -r kubernetes config "$MYTMPDIR"
+  cd "$MYTMPDIR/kubernetes"
   kustomize edit set image "plugin-barman-cloud=$operator_image"
   kustomize edit set secret plugin-barman-cloud "--from-literal=SIDECAR_IMAGE=$instance_image"
+  kubectl apply -k .
 )
-
-# Now we deploy the plugin inside the `cnpg-system` workspace
-kubectl apply -k kubernetes/