fix: classify WAL restore errors with precise gRPC statuses

Map each restorer sentinel to the gRPC status that best reflects whether
the operator should retry:

  - ErrWALNotFound      -> NotFound        (terminal)
  - ErrInvalidWalName   -> InvalidArgument (terminal)
  - ErrConnectivity     -> Unavailable     (retry)
  - ErrGeneric          -> Unavailable     (retry)
  - anything else       -> Internal        (terminal)

Previously every non-NotFound failure was returned verbatim, leaving the
operator unable to tell transient blips apart from terminal conditions
like a malformed WAL name.

Signed-off-by: Leonardo Cecchi <leonardo.cecchi@enterprisedb.com>
This commit is contained in:
Leonardo Cecchi 2026-05-27 10:17:54 +02:00
parent d244faa5b2
commit 40f7ce24ab
4 changed files with 75 additions and 16 deletions

6
go.mod
View File

@ -9,7 +9,7 @@ require (
github.com/cloudnative-pg/cloudnative-pg v1.29.1 github.com/cloudnative-pg/cloudnative-pg v1.29.1
github.com/cloudnative-pg/cnpg-i v0.5.0 github.com/cloudnative-pg/cnpg-i v0.5.0
github.com/cloudnative-pg/cnpg-i-machinery v0.4.2 github.com/cloudnative-pg/cnpg-i-machinery v0.4.2
github.com/cloudnative-pg/machinery v0.4.0 github.com/cloudnative-pg/machinery v0.5.0
github.com/onsi/ginkgo/v2 v2.29.0 github.com/onsi/ginkgo/v2 v2.29.0
github.com/onsi/gomega v1.41.0 github.com/onsi/gomega v1.41.0
github.com/spf13/cobra v1.10.2 github.com/spf13/cobra v1.10.2
@ -114,7 +114,7 @@ require (
golang.org/x/net v0.53.0 // indirect golang.org/x/net v0.53.0 // indirect
golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect
golang.org/x/sync v0.20.0 // indirect golang.org/x/sync v0.20.0 // indirect
golang.org/x/sys v0.43.0 // indirect golang.org/x/sys v0.45.0 // indirect
golang.org/x/term v0.42.0 // indirect golang.org/x/term v0.42.0 // indirect
golang.org/x/text v0.36.0 // indirect golang.org/x/text v0.36.0 // indirect
golang.org/x/time v0.15.0 // indirect golang.org/x/time v0.15.0 // indirect
@ -137,3 +137,5 @@ require (
sigs.k8s.io/structured-merge-diff/v6 v6.4.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.4.0 // indirect
sigs.k8s.io/yaml v1.6.0 // indirect sigs.k8s.io/yaml v1.6.0 // indirect
) )
replace github.com/cloudnative-pg/barman-cloud => github.com/leonardoce/barman-cloud v0.0.0-20260527084420-d2ef1c114ae8

12
go.sum
View File

@ -20,16 +20,14 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cloudnative-pg/api v1.29.1 h1:FWr8S7EQeOfdhYXyr2cof8wUXLARZiiQt5Qa6ltED7w= github.com/cloudnative-pg/api v1.29.1 h1:FWr8S7EQeOfdhYXyr2cof8wUXLARZiiQt5Qa6ltED7w=
github.com/cloudnative-pg/api v1.29.1/go.mod h1:QtWF3yzSvIfORMHaSkPAk/o3bhCJwEHJgN3riyRiz3o= github.com/cloudnative-pg/api v1.29.1/go.mod h1:QtWF3yzSvIfORMHaSkPAk/o3bhCJwEHJgN3riyRiz3o=
github.com/cloudnative-pg/barman-cloud v0.5.1 h1:vjkXrrxo2DQXHT9u9usqhtaHiPZ/lTfDVs/pIWYTepQ=
github.com/cloudnative-pg/barman-cloud v0.5.1/go.mod h1:XPc5IUFP1y4cZX1sg+Pd8j9V4tmUEVnv3BGCpfQOOg8=
github.com/cloudnative-pg/cloudnative-pg v1.29.1 h1:ZNEt1TMlnQKXI1kho2UqQuqdfvIvjGln4kN7C1lsmGA= github.com/cloudnative-pg/cloudnative-pg v1.29.1 h1:ZNEt1TMlnQKXI1kho2UqQuqdfvIvjGln4kN7C1lsmGA=
github.com/cloudnative-pg/cloudnative-pg v1.29.1/go.mod h1:Sbgx9jVmkle4/gR2U5JHrzDd74sRPOBHDtPkvncg5v8= github.com/cloudnative-pg/cloudnative-pg v1.29.1/go.mod h1:Sbgx9jVmkle4/gR2U5JHrzDd74sRPOBHDtPkvncg5v8=
github.com/cloudnative-pg/cnpg-i v0.5.0 h1:/TOzpNT6cwNgrpftTtrnLKdoHgMwd+88vZgXjlVgXeE= github.com/cloudnative-pg/cnpg-i v0.5.0 h1:/TOzpNT6cwNgrpftTtrnLKdoHgMwd+88vZgXjlVgXeE=
github.com/cloudnative-pg/cnpg-i v0.5.0/go.mod h1:7Gh4+UzhBpGhr4DreB1GN9wGYfvxwXCXZUyVt3zE/3I= github.com/cloudnative-pg/cnpg-i v0.5.0/go.mod h1:7Gh4+UzhBpGhr4DreB1GN9wGYfvxwXCXZUyVt3zE/3I=
github.com/cloudnative-pg/cnpg-i-machinery v0.4.2 h1:0reS9MtyLYINHXQ/MfxJ9jp39hhBf8e3Qdj+T5Nsq6I= github.com/cloudnative-pg/cnpg-i-machinery v0.4.2 h1:0reS9MtyLYINHXQ/MfxJ9jp39hhBf8e3Qdj+T5Nsq6I=
github.com/cloudnative-pg/cnpg-i-machinery v0.4.2/go.mod h1:gvrKabgxXq0zGthXGucemDdsxakLEQDMxn43M4HLW30= github.com/cloudnative-pg/cnpg-i-machinery v0.4.2/go.mod h1:gvrKabgxXq0zGthXGucemDdsxakLEQDMxn43M4HLW30=
github.com/cloudnative-pg/machinery v0.4.0 h1:3sfqrBptH4QQSVB4g10Z+7aiQRnh4g+6AsqsK0ibKaQ= github.com/cloudnative-pg/machinery v0.5.0 h1:hhTnkzn+AiN3NmbjCQ6RXj5rfqV3K6arzq6kdXAzcnQ=
github.com/cloudnative-pg/machinery v0.4.0/go.mod h1:OIwaTYAnLv8PmBmBvEf0BvMK2JBX6J+naTMg9UgV1FQ= github.com/cloudnative-pg/machinery v0.5.0/go.mod h1:uuFjqBUjWn0a9uvAk1ixTSzPM0PrjaS+QiKLOIBqLm4=
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -143,6 +141,8 @@ github.com/kubernetes-csi/external-snapshotter/client/v8 v8.4.0 h1:bMqrb3UHgHbP+
github.com/kubernetes-csi/external-snapshotter/client/v8 v8.4.0/go.mod h1:E3vdYxHj2C2q6qo8/Da4g7P+IcwqRZyy3gJBzYybV9Y= github.com/kubernetes-csi/external-snapshotter/client/v8 v8.4.0/go.mod h1:E3vdYxHj2C2q6qo8/Da4g7P+IcwqRZyy3gJBzYybV9Y=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/leonardoce/barman-cloud v0.0.0-20260527084420-d2ef1c114ae8 h1:svNVHqtp2a7FfxuLPTm7bs9esj/tAp4arYY/8hvK3no=
github.com/leonardoce/barman-cloud v0.0.0-20260527084420-d2ef1c114ae8/go.mod h1:DACs5rdlMIQ7mUOpJVBRTrbf+d7OGyMOLsFUEMJ2JV8=
github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ= github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ=
github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=
github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo=
@ -275,8 +275,8 @@ golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY=
golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY=
golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=

View File

@ -35,10 +35,50 @@ var ErrEndOfWALStreamReached = status.Errorf(codes.OutOfRange, "end of WAL reach
var ErrMissingPermissions = status.Errorf(codes.FailedPrecondition, var ErrMissingPermissions = status.Errorf(codes.FailedPrecondition,
"no permission to download the backup credentials, retrying") "no permission to download the backup credentials, retrying")
// newWALNotFoundError returns a error that states that a // newWALNotFoundError returns an error indicating that the
// certain WAL file has not been found. This error is // requested WAL file is not present in the object store.
// compatible with GRPC status codes, resulting in a 404 // It carries a gRPC NotFound status so the operator can
// being used as a response code. // treat it as a terminal condition.
func newWALNotFoundError(walName string) error { func newWALNotFoundError(walName string) error {
return status.Errorf(codes.NotFound, "wal %q not found", walName) return status.Errorf(codes.NotFound, "wal %q not found", walName)
} }
// newUnavailableError returns an error indicating that
// downloading the given WAL file failed for a transient
// reason (e.g. a connectivity blip). It carries a gRPC
// Unavailable status so the operator will retry the
// request.
func newUnavailableError(walName string, err error) error {
return status.Errorf(
codes.Unavailable,
"transient error while downloading %q: %s",
walName,
err.Error(),
)
}
// newInvalidWALNameError returns an error indicating that
// the requested WAL name is not valid. It carries a gRPC
// InvalidArgument status so the operator treats it as a
// terminal condition.
func newInvalidWALNameError(walName string, err error) error {
return status.Errorf(
codes.InvalidArgument,
"invalid WAL name %q: %s",
walName,
err.Error(),
)
}
// newInternalWALRestoreError returns an error indicating
// that downloading the given WAL file failed for an
// unclassified reason. It carries a gRPC Internal status
// so the operator treats it as a terminal condition.
func newInternalWALRestoreError(walName string, err error) error {
return status.Errorf(
codes.Internal,
"internal error while downloading %q: %s",
walName,
err.Error(),
)
}

View File

@ -365,11 +365,28 @@ func (w WALServiceImplementation) restoreFromBarmanObjectStore(
// is the one that PostgreSQL has requested to restore. // is the one that PostgreSQL has requested to restore.
// The failure has already been logged in walRestorer.RestoreList method // The failure has already been logged in walRestorer.RestoreList method
if walStatus[0].Err != nil { if walStatus[0].Err != nil {
if errors.Is(walStatus[0].Err, barmanRestorer.ErrWALNotFound) { walName := walStatus[0].WalName
return newWALNotFoundError(walStatus[0].WalName) walErr := walStatus[0].Err
switch {
case errors.Is(walErr, barmanRestorer.ErrWALNotFound):
// A missing WAL is a terminal condition: surface it as a
// gRPC NotFound so the operator stops retrying.
return newWALNotFoundError(walName)
case errors.Is(walErr, barmanRestorer.ErrInvalidWalName):
// A malformed WAL name will never succeed on retry.
return newInvalidWALNameError(walName, walErr)
case errors.Is(walErr, barmanRestorer.ErrConnectivity),
errors.Is(walErr, barmanRestorer.ErrGeneric):
// Connectivity and generic barman-cloud failures are
// considered transient: surface them as gRPC Unavailable
// so the operator retries the download.
return newUnavailableError(walName, walErr)
default:
// Unrecognized exit codes and unexpected failures (e.g. the
// barman-cloud command could not be executed) are surfaced
// as gRPC Internal so the operator treats them as terminal.
return newInternalWALRestoreError(walName, walErr)
} }
return walStatus[0].Err
} }
// We skip this step if streaming connection is not available // We skip this step if streaming connection is not available