fix: classify WAL restore errors with precise gRPC statuses

Map each restorer sentinel to the gRPC status that best reflects whether
the operator should retry:

  - ErrWALNotFound      -> NotFound        (terminal)
  - ErrInvalidWalName   -> InvalidArgument (terminal)
  - ErrConnectivity     -> Unavailable     (retry)
  - ErrGeneric          -> Unavailable     (retry)
  - anything else       -> Internal        (terminal)

Previously every non-NotFound failure was returned verbatim, leaving the
operator unable to tell transient blips apart from terminal conditions
like a malformed WAL name.

Signed-off-by: Leonardo Cecchi <leonardo.cecchi@enterprisedb.com>
This commit is contained in:
Leonardo Cecchi 2026-05-27 10:17:54 +02:00
parent 8e2a917153
commit d30bbdfae3
4 changed files with 69 additions and 10 deletions

2
go.mod
View File

@ -137,3 +137,5 @@ require (
sigs.k8s.io/structured-merge-diff/v6 v6.4.0 // indirect
sigs.k8s.io/yaml v1.6.0 // indirect
)
replace github.com/cloudnative-pg/barman-cloud => github.com/leonardoce/barman-cloud v0.0.0-20260527084420-d2ef1c114ae8

4
go.sum
View File

@ -20,8 +20,6 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cloudnative-pg/api v1.29.1 h1:FWr8S7EQeOfdhYXyr2cof8wUXLARZiiQt5Qa6ltED7w=
github.com/cloudnative-pg/api v1.29.1/go.mod h1:QtWF3yzSvIfORMHaSkPAk/o3bhCJwEHJgN3riyRiz3o=
github.com/cloudnative-pg/barman-cloud v0.5.1 h1:vjkXrrxo2DQXHT9u9usqhtaHiPZ/lTfDVs/pIWYTepQ=
github.com/cloudnative-pg/barman-cloud v0.5.1/go.mod h1:XPc5IUFP1y4cZX1sg+Pd8j9V4tmUEVnv3BGCpfQOOg8=
github.com/cloudnative-pg/cloudnative-pg v1.29.1 h1:ZNEt1TMlnQKXI1kho2UqQuqdfvIvjGln4kN7C1lsmGA=
github.com/cloudnative-pg/cloudnative-pg v1.29.1/go.mod h1:Sbgx9jVmkle4/gR2U5JHrzDd74sRPOBHDtPkvncg5v8=
github.com/cloudnative-pg/cnpg-i v0.5.0 h1:/TOzpNT6cwNgrpftTtrnLKdoHgMwd+88vZgXjlVgXeE=
@ -143,6 +141,8 @@ github.com/kubernetes-csi/external-snapshotter/client/v8 v8.4.0 h1:bMqrb3UHgHbP+
github.com/kubernetes-csi/external-snapshotter/client/v8 v8.4.0/go.mod h1:E3vdYxHj2C2q6qo8/Da4g7P+IcwqRZyy3gJBzYybV9Y=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/leonardoce/barman-cloud v0.0.0-20260527084420-d2ef1c114ae8 h1:svNVHqtp2a7FfxuLPTm7bs9esj/tAp4arYY/8hvK3no=
github.com/leonardoce/barman-cloud v0.0.0-20260527084420-d2ef1c114ae8/go.mod h1:DACs5rdlMIQ7mUOpJVBRTrbf+d7OGyMOLsFUEMJ2JV8=
github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ=
github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=
github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo=

View File

@ -35,10 +35,50 @@ var ErrEndOfWALStreamReached = status.Errorf(codes.OutOfRange, "end of WAL reach
var ErrMissingPermissions = status.Errorf(codes.FailedPrecondition,
"no permission to download the backup credentials, retrying")
// newWALNotFoundError returns a error that states that a
// certain WAL file has not been found. This error is
// compatible with GRPC status codes, resulting in a 404
// being used as a response code.
// newWALNotFoundError returns an error indicating that the
// requested WAL file is not present in the object store.
// It carries a gRPC NotFound status so the operator can
// treat it as a terminal condition.
func newWALNotFoundError(walName string) error {
return status.Errorf(codes.NotFound, "wal %q not found", walName)
}
// newUnavailableError returns an error indicating that
// downloading the given WAL file failed for a transient
// reason (e.g. a connectivity blip). It carries a gRPC
// Unavailable status so the operator will retry the
// request.
func newUnavailableError(walName string, err error) error {
return status.Errorf(
codes.Unavailable,
"transient error while downloading %q: %s",
walName,
err.Error(),
)
}
// newInvalidWALNameError returns an error indicating that
// the requested WAL name is not valid. It carries a gRPC
// InvalidArgument status so the operator treats it as a
// terminal condition.
func newInvalidWALNameError(walName string, err error) error {
return status.Errorf(
codes.InvalidArgument,
"invalid WAL name %q: %s",
walName,
err.Error(),
)
}
// newInternalWALRestoreError returns an error indicating
// that downloading the given WAL file failed for an
// unclassified reason. It carries a gRPC Internal status
// so the operator treats it as a terminal condition.
func newInternalWALRestoreError(walName string, err error) error {
return status.Errorf(
codes.Internal,
"internal error while downloading %q: %s",
walName,
err.Error(),
)
}

View File

@ -365,11 +365,28 @@ func (w WALServiceImplementation) restoreFromBarmanObjectStore(
// is the one that PostgreSQL has requested to restore.
// The failure has already been logged in walRestorer.RestoreList method
if walStatus[0].Err != nil {
if errors.Is(walStatus[0].Err, barmanRestorer.ErrWALNotFound) {
return newWALNotFoundError(walStatus[0].WalName)
walName := walStatus[0].WalName
walErr := walStatus[0].Err
switch {
case errors.Is(walErr, barmanRestorer.ErrWALNotFound):
// A missing WAL is a terminal condition: surface it as a
// gRPC NotFound so the operator stops retrying.
return newWALNotFoundError(walName)
case errors.Is(walErr, barmanRestorer.ErrInvalidWalName):
// A malformed WAL name will never succeed on retry.
return newInvalidWALNameError(walName, walErr)
case errors.Is(walErr, barmanRestorer.ErrConnectivity),
errors.Is(walErr, barmanRestorer.ErrGeneric):
// Connectivity and generic barman-cloud failures are
// considered transient: surface them as gRPC Unavailable
// so the operator retries the download.
return newUnavailableError(walName, walErr)
default:
// Unrecognized exit codes and unexpected failures (e.g. the
// barman-cloud command could not be executed) are surfaced
// as gRPC Internal so the operator treats them as terminal.
return newInternalWALRestoreError(walName, walErr)
}
return walStatus[0].Err
}
// We skip this step if streaming connection is not available