diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 0f2bfdf..d4e553f 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -33,6 +33,7 @@ import ( internalClient "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/client" "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/deployment" + "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/diagnostics" "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/e2etestenv" "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/kustomize" @@ -131,6 +132,28 @@ var _ = SynchronizedBeforeSuite(func(ctx SpecContext) []byte { logFlags.ConfigureLogging() }) +// The ephemeral cluster the suite runs against is torn down right after this +// process exits, so this is the only chance to capture the CloudNativePG +// operator and barman-cloud plugin logs for a failed run. +var _ = ReportAfterSuite("dump cnpg-system diagnostics on failure", func(report Report) { + if report.SuiteSucceeded { + return + } + + cl, _, err := internalClient.NewClient() + if err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "failed to create Kubernetes client for diagnostics: %v\n", err) + return + } + clientSet, _, err := internalClient.NewClientSet() + if err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "failed to create Kubernetes clientset for diagnostics: %v\n", err) + return + } + + diagnostics.DumpOperatorNamespace(context.Background(), cl, clientSet, "cnpg-system") +}) + // Run e2e tests using the Ginkgo runner. func TestE2E(t *testing.T) { RegisterFailHandler(Fail) diff --git a/test/e2e/internal/diagnostics/diagnostics.go b/test/e2e/internal/diagnostics/diagnostics.go new file mode 100644 index 0000000..34f4ee0 --- /dev/null +++ b/test/e2e/internal/diagnostics/diagnostics.go @@ -0,0 +1,209 @@ +/* +Copyright © contributors to CloudNativePG, established as +CloudNativePG a Series of LF Projects, LLC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +*/ + +// Package diagnostics provides helpers to capture the state of a namespace +// when an e2e spec fails, so the root cause can be inspected from the CI +// logs without needing to reproduce the failure locally. +package diagnostics + +import ( + "context" + "fmt" + "io" + "slices" + "sort" + + cloudnativepgv1 "github.com/cloudnative-pg/api/pkg/api/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/client" + + . "github.com/onsi/ginkgo/v2" +) + +// pluginContainerName is the name of the barman-cloud plugin sidecar injected +// into every PostgreSQL pod, see internal/cnpgi/operator/lifecycle.go. +const pluginContainerName = "plugin-barman-cloud" + +// tailLines is the number of trailing log lines fetched from each container. +const tailLines = 200 + +// DumpNamespace prints diagnostic information about the given test namespace +// to the GinkgoWriter: namespace Events, Backup and Cluster statuses, pod +// container statuses, and the tail of the postgres/plugin container logs. +// It is a no-op if the current spec has not failed, so it is safe to call +// unconditionally from an AfterEach, before the namespace is torn down. +func DumpNamespace(ctx context.Context, cl client.Client, clientSet *kubernetes.Clientset, namespaceName string) { + if !CurrentSpecReport().Failed() { + return + } + + dumpNamespace(ctx, cl, clientSet, namespaceName, "postgres", pluginContainerName) +} + +// DumpOperatorNamespace prints the same diagnostics as DumpNamespace, but for +// the cnpg-system namespace the CloudNativePG operator and the barman-cloud +// plugin run in, dumping the logs of every container in it. Unlike +// DumpNamespace it always runs when called: the caller (typically a +// ReportAfterSuite, since cnpg-system isn't torn down per-spec) is expected +// to only call it once the aggregated suite report shows a failure. +func DumpOperatorNamespace( + ctx context.Context, + cl client.Client, + clientSet *kubernetes.Clientset, + namespaceName string, +) { + dumpNamespace(ctx, cl, clientSet, namespaceName) +} + +func dumpNamespace( + ctx context.Context, + cl client.Client, + clientSet *kubernetes.Clientset, + namespaceName string, + containerNames ...string, +) { + fmt.Fprintf(GinkgoWriter, "\n::group::Diagnostics for namespace %q\n", namespaceName) + defer fmt.Fprintln(GinkgoWriter, "::endgroup::") + + dumpEvents(ctx, cl, namespaceName) + dumpBackups(ctx, cl, namespaceName) + dumpClusters(ctx, cl, namespaceName) + dumpPods(ctx, cl, clientSet, namespaceName, containerNames) +} + +func dumpEvents(ctx context.Context, cl client.Client, namespaceName string) { + var events corev1.EventList + if err := cl.List(ctx, &events, client.InNamespace(namespaceName)); err != nil { + fmt.Fprintf(GinkgoWriter, "failed to list events in %q: %v\n", namespaceName, err) + return + } + + sort.Slice(events.Items, func(i, j int) bool { + return events.Items[i].LastTimestamp.Before(&events.Items[j].LastTimestamp) + }) + + fmt.Fprintf(GinkgoWriter, "-- Events (%d) --\n", len(events.Items)) + for _, event := range events.Items { + fmt.Fprintf(GinkgoWriter, "[%s] %s/%s %s: %s\n", + event.LastTimestamp.Format("15:04:05"), + event.InvolvedObject.Kind, event.InvolvedObject.Name, + event.Reason, event.Message) + } +} + +func dumpBackups(ctx context.Context, cl client.Client, namespaceName string) { + var backups cloudnativepgv1.BackupList + if err := cl.List(ctx, &backups, client.InNamespace(namespaceName)); err != nil { + fmt.Fprintf(GinkgoWriter, "failed to list backups in %q: %v\n", namespaceName, err) + return + } + + fmt.Fprintf(GinkgoWriter, "-- Backups (%d) --\n", len(backups.Items)) + for _, backup := range backups.Items { + fmt.Fprintf(GinkgoWriter, "%s: phase=%s error=%q commandError=%q\n", + backup.Name, backup.Status.Phase, backup.Status.Error, backup.Status.CommandError) + } +} + +func dumpClusters(ctx context.Context, cl client.Client, namespaceName string) { + var clusters cloudnativepgv1.ClusterList + if err := cl.List(ctx, &clusters, client.InNamespace(namespaceName)); err != nil { + fmt.Fprintf(GinkgoWriter, "failed to list clusters in %q: %v\n", namespaceName, err) + return + } + + fmt.Fprintf(GinkgoWriter, "-- Clusters (%d) --\n", len(clusters.Items)) + for _, cluster := range clusters.Items { + fmt.Fprintf(GinkgoWriter, "%s: phase=%s reason=%q readyInstances=%d/%d\n", + cluster.Name, cluster.Status.Phase, cluster.Status.PhaseReason, + cluster.Status.ReadyInstances, cluster.Spec.Instances) + } +} + +// dumpPods prints every pod's container statuses in namespaceName, and the +// tail of the log of each container whose name is in containerNames (or +// every container, if containerNames is empty). +func dumpPods( + ctx context.Context, + cl client.Client, + clientSet *kubernetes.Clientset, + namespaceName string, + containerNames []string, +) { + var pods corev1.PodList + if err := cl.List(ctx, &pods, client.InNamespace(namespaceName)); err != nil { + fmt.Fprintf(GinkgoWriter, "failed to list pods in %q: %v\n", namespaceName, err) + return + } + + fmt.Fprintf(GinkgoWriter, "-- Pods (%d) --\n", len(pods.Items)) + for _, pod := range pods.Items { + fmt.Fprintf(GinkgoWriter, "%s: phase=%s\n", pod.Name, pod.Status.Phase) + for _, cs := range pod.Status.ContainerStatuses { + fmt.Fprintf(GinkgoWriter, " container %s: ready=%t restarts=%d state=%s\n", + cs.Name, cs.Ready, cs.RestartCount, containerStateString(cs.State)) + + if len(containerNames) == 0 || slices.Contains(containerNames, cs.Name) { + dumpContainerLog(ctx, clientSet, namespaceName, pod.Name, cs.Name) + } + } + } +} + +func containerStateString(state corev1.ContainerState) string { + switch { + case state.Waiting != nil: + return fmt.Sprintf("waiting(%s: %s)", state.Waiting.Reason, state.Waiting.Message) + case state.Running != nil: + return fmt.Sprintf("running(since %s)", state.Running.StartedAt) + case state.Terminated != nil: + return fmt.Sprintf("terminated(exitCode=%d reason=%s: %s)", + state.Terminated.ExitCode, state.Terminated.Reason, state.Terminated.Message) + default: + return "unknown" + } +} + +func dumpContainerLog( + ctx context.Context, + clientSet *kubernetes.Clientset, + namespaceName, podName, containerName string, +) { + tail := int64(tailLines) + req := clientSet.CoreV1().Pods(namespaceName).GetLogs(podName, &corev1.PodLogOptions{ + Container: containerName, + TailLines: &tail, + }) + + stream, err := req.Stream(ctx) + if err != nil { + fmt.Fprintf(GinkgoWriter, " failed to fetch logs for %s/%s: %v\n", podName, containerName, err) + return + } + defer stream.Close() + + logs, err := io.ReadAll(stream) + if err != nil { + fmt.Fprintf(GinkgoWriter, " failed to read logs for %s/%s: %v\n", podName, containerName, err) + return + } + + fmt.Fprintf(GinkgoWriter, " -- last %d lines of %s/%s --\n%s\n", tailLines, podName, containerName, logs) +} diff --git a/test/e2e/internal/tests/backup/backup_restore.go b/test/e2e/internal/tests/backup/backup_restore.go index 09892d0..ff8aaa4 100644 --- a/test/e2e/internal/tests/backup/backup_restore.go +++ b/test/e2e/internal/tests/backup/backup_restore.go @@ -26,11 +26,13 @@ import ( v1 "github.com/cloudnative-pg/api/pkg/api/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" internalClient "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/client" internalCluster "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/cluster" "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/command" + "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/diagnostics" nmsp "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/namespace" . "github.com/onsi/ginkgo/v2" @@ -40,14 +42,18 @@ import ( var _ = Describe("Backup and restore", func() { var namespace *corev1.Namespace var cl client.Client + var diagClientSet *kubernetes.Clientset BeforeEach(func(ctx SpecContext) { var err error cl, _, err = internalClient.NewClient() Expect(err).NotTo(HaveOccurred()) + diagClientSet, _, err = internalClient.NewClientSet() + Expect(err).NotTo(HaveOccurred()) namespace, err = nmsp.CreateUniqueNamespace(ctx, cl, "backup-restore") Expect(err).NotTo(HaveOccurred()) }) AfterEach(func(ctx SpecContext) { + diagnostics.DumpNamespace(ctx, cl, diagClientSet, namespace.Name) Expect(cl.Delete(ctx, namespace)).To(Succeed()) }) diff --git a/test/e2e/internal/tests/credentialrotation/credential_rotation.go b/test/e2e/internal/tests/credentialrotation/credential_rotation.go index f530259..a920561 100644 --- a/test/e2e/internal/tests/credentialrotation/credential_rotation.go +++ b/test/e2e/internal/tests/credentialrotation/credential_rotation.go @@ -27,6 +27,7 @@ import ( rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -34,6 +35,7 @@ import ( "github.com/cloudnative-pg/plugin-barman-cloud/internal/cnpgi/operator/specs" internalClient "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/client" internalCluster "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/cluster" + "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/diagnostics" nmsp "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/namespace" "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/objectstore" @@ -51,16 +53,20 @@ const ( var _ = Describe("Credential rotation", func() { var namespace *corev1.Namespace var cl client.Client + var diagClientSet *kubernetes.Clientset BeforeEach(func(ctx SpecContext) { var err error cl, _, err = internalClient.NewClient() Expect(err).NotTo(HaveOccurred()) + diagClientSet, _, err = internalClient.NewClientSet() + Expect(err).NotTo(HaveOccurred()) namespace, err = nmsp.CreateUniqueNamespace(ctx, cl, "cred-rotation") Expect(err).NotTo(HaveOccurred()) }) AfterEach(func(ctx SpecContext) { + diagnostics.DumpNamespace(ctx, cl, diagClientSet, namespace.Name) Expect(cl.Delete(ctx, namespace)).To(Succeed()) }) diff --git a/test/e2e/internal/tests/replicacluster/replica_cluster.go b/test/e2e/internal/tests/replicacluster/replica_cluster.go index 8888334..f33208c 100644 --- a/test/e2e/internal/tests/replicacluster/replica_cluster.go +++ b/test/e2e/internal/tests/replicacluster/replica_cluster.go @@ -27,12 +27,14 @@ import ( cloudnativepgv1 "github.com/cloudnative-pg/api/pkg/api/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" internalClient "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/client" cluster2 "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/cluster" "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/command" + "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/diagnostics" nmsp "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/namespace" . "github.com/onsi/ginkgo/v2" @@ -42,14 +44,18 @@ import ( var _ = Describe("Replica cluster", func() { var namespace *corev1.Namespace var cl client.Client + var diagClientSet *kubernetes.Clientset BeforeEach(func(ctx SpecContext) { var err error cl, _, err = internalClient.NewClient() Expect(err).NotTo(HaveOccurred()) + diagClientSet, _, err = internalClient.NewClientSet() + Expect(err).NotTo(HaveOccurred()) namespace, err = nmsp.CreateUniqueNamespace(ctx, cl, "replica-cluster") Expect(err).NotTo(HaveOccurred()) }) AfterEach(func(ctx SpecContext) { + diagnostics.DumpNamespace(ctx, cl, diagClientSet, namespace.Name) Expect(cl.Delete(ctx, namespace)).To(Succeed()) }) DescribeTable("can switchover to a replica cluster", diff --git a/test/e2e/internal/tests/walrestore/walrestore.go b/test/e2e/internal/tests/walrestore/walrestore.go index 287748f..e4b4e94 100644 --- a/test/e2e/internal/tests/walrestore/walrestore.go +++ b/test/e2e/internal/tests/walrestore/walrestore.go @@ -38,6 +38,7 @@ import ( internalCluster "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/cluster" "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/command" "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/deployment" + "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/diagnostics" nmsp "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/namespace" . "github.com/onsi/ginkgo/v2" @@ -128,6 +129,7 @@ var _ = Describe("Parallel WAL restore", func() { }) AfterEach(func(ctx SpecContext) { + diagnostics.DumpNamespace(ctx, cl, clientSet, namespace.Name) Expect(cl.Delete(ctx, namespace)).To(Succeed()) })