code.dwrz.net

Go monorepo.
Log | Files | Refs

standard.go (7979B)

      1 package retry
      2 
      3 import (
      4 	"context"
      5 	"fmt"
      6 	"time"
      7 
      8 	"github.com/aws/aws-sdk-go-v2/aws/ratelimit"
      9 )
     10 
     11 // BackoffDelayer provides the interface for determining the delay to before
     12 // another request attempt, that previously failed.
     13 type BackoffDelayer interface {
     14 	BackoffDelay(attempt int, err error) (time.Duration, error)
     15 }
     16 
     17 // BackoffDelayerFunc provides a wrapper around a function to determine the
     18 // backoff delay of an attempt retry.
     19 type BackoffDelayerFunc func(int, error) (time.Duration, error)
     20 
     21 // BackoffDelay returns the delay before attempt to retry a request.
     22 func (fn BackoffDelayerFunc) BackoffDelay(attempt int, err error) (time.Duration, error) {
     23 	return fn(attempt, err)
     24 }
     25 
     26 const (
     27 	// DefaultMaxAttempts is the maximum of attempts for an API request
     28 	DefaultMaxAttempts int = 3
     29 
     30 	// DefaultMaxBackoff is the maximum back off delay between attempts
     31 	DefaultMaxBackoff time.Duration = 20 * time.Second
     32 )
     33 
     34 // Default retry token quota values.
     35 const (
     36 	DefaultRetryRateTokens  uint = 500
     37 	DefaultRetryCost        uint = 5
     38 	DefaultRetryTimeoutCost uint = 10
     39 	DefaultNoRetryIncrement uint = 1
     40 )
     41 
     42 // DefaultRetryableHTTPStatusCodes is the default set of HTTP status codes the SDK
     43 // should consider as retryable errors.
     44 var DefaultRetryableHTTPStatusCodes = map[int]struct{}{
     45 	500: {},
     46 	502: {},
     47 	503: {},
     48 	504: {},
     49 }
     50 
     51 // DefaultRetryableErrorCodes provides the set of API error codes that should
     52 // be retried.
     53 var DefaultRetryableErrorCodes = map[string]struct{}{
     54 	"RequestTimeout":          {},
     55 	"RequestTimeoutException": {},
     56 }
     57 
     58 // DefaultThrottleErrorCodes provides the set of API error codes that are
     59 // considered throttle errors.
     60 var DefaultThrottleErrorCodes = map[string]struct{}{
     61 	"Throttling":                             {},
     62 	"ThrottlingException":                    {},
     63 	"ThrottledException":                     {},
     64 	"RequestThrottledException":              {},
     65 	"TooManyRequestsException":               {},
     66 	"ProvisionedThroughputExceededException": {},
     67 	"TransactionInProgressException":         {},
     68 	"RequestLimitExceeded":                   {},
     69 	"BandwidthLimitExceeded":                 {},
     70 	"LimitExceededException":                 {},
     71 	"RequestThrottled":                       {},
     72 	"SlowDown":                               {},
     73 	"PriorRequestNotComplete":                {},
     74 	"EC2ThrottledException":                  {},
     75 }
     76 
     77 // DefaultRetryables provides the set of retryable checks that are used by
     78 // default.
     79 var DefaultRetryables = []IsErrorRetryable{
     80 	NoRetryCanceledError{},
     81 	RetryableError{},
     82 	RetryableConnectionError{},
     83 	RetryableHTTPStatusCode{
     84 		Codes: DefaultRetryableHTTPStatusCodes,
     85 	},
     86 	RetryableErrorCode{
     87 		Codes: DefaultRetryableErrorCodes,
     88 	},
     89 	RetryableErrorCode{
     90 		Codes: DefaultThrottleErrorCodes,
     91 	},
     92 }
     93 
     94 // DefaultTimeouts provides the set of timeout checks that are used by default.
     95 var DefaultTimeouts = []IsErrorTimeout{
     96 	TimeouterError{},
     97 }
     98 
     99 // StandardOptions provides the functional options for configuring the standard
    100 // retryable, and delay behavior.
    101 type StandardOptions struct {
    102 	// Maximum number of attempts that should be made.
    103 	MaxAttempts int
    104 
    105 	// MaxBackoff duration between retried attempts.
    106 	MaxBackoff time.Duration
    107 
    108 	// Provides the backoff strategy the retryer will use to determine the
    109 	// delay between retry attempts.
    110 	Backoff BackoffDelayer
    111 
    112 	// Set of strategies to determine if the attempt should be retried based on
    113 	// the error response received.
    114 	//
    115 	// It is safe to append to this list in NewStandard's functional options.
    116 	Retryables []IsErrorRetryable
    117 
    118 	// Set of strategies to determine if the attempt failed due to a timeout
    119 	// error.
    120 	//
    121 	// It is safe to append to this list in NewStandard's functional options.
    122 	Timeouts []IsErrorTimeout
    123 
    124 	// Provides the rate limiting strategy for rate limiting attempt retries
    125 	// across all attempts the retryer is being used with.
    126 	RateLimiter RateLimiter
    127 
    128 	// The cost to deduct from the RateLimiter's token bucket per retry.
    129 	RetryCost uint
    130 
    131 	// The cost to deduct from the RateLimiter's token bucket per retry caused
    132 	// by timeout error.
    133 	RetryTimeoutCost uint
    134 
    135 	// The cost to payback to the RateLimiter's token bucket for successful
    136 	// attempts.
    137 	NoRetryIncrement uint
    138 }
    139 
    140 // RateLimiter provides the interface for limiting the rate of attempt retries
    141 // allowed by the retryer.
    142 type RateLimiter interface {
    143 	GetToken(ctx context.Context, cost uint) (releaseToken func() error, err error)
    144 	AddTokens(uint) error
    145 }
    146 
    147 // Standard is the standard retry pattern for the SDK. It uses a set of
    148 // retryable checks to determine of the failed attempt should be retried, and
    149 // what retry delay should be used.
    150 type Standard struct {
    151 	options StandardOptions
    152 
    153 	timeout   IsErrorTimeout
    154 	retryable IsErrorRetryable
    155 	backoff   BackoffDelayer
    156 }
    157 
    158 // NewStandard initializes a standard retry behavior with defaults that can be
    159 // overridden via functional options.
    160 func NewStandard(fnOpts ...func(*StandardOptions)) *Standard {
    161 	o := StandardOptions{
    162 		MaxAttempts: DefaultMaxAttempts,
    163 		MaxBackoff:  DefaultMaxBackoff,
    164 		Retryables:  append([]IsErrorRetryable{}, DefaultRetryables...),
    165 		Timeouts:    append([]IsErrorTimeout{}, DefaultTimeouts...),
    166 
    167 		RateLimiter:      ratelimit.NewTokenRateLimit(DefaultRetryRateTokens),
    168 		RetryCost:        DefaultRetryCost,
    169 		RetryTimeoutCost: DefaultRetryTimeoutCost,
    170 		NoRetryIncrement: DefaultNoRetryIncrement,
    171 	}
    172 	for _, fn := range fnOpts {
    173 		fn(&o)
    174 	}
    175 	if o.MaxAttempts <= 0 {
    176 		o.MaxAttempts = DefaultMaxAttempts
    177 	}
    178 
    179 	backoff := o.Backoff
    180 	if backoff == nil {
    181 		backoff = NewExponentialJitterBackoff(o.MaxBackoff)
    182 	}
    183 
    184 	return &Standard{
    185 		options:   o,
    186 		backoff:   backoff,
    187 		retryable: IsErrorRetryables(o.Retryables),
    188 		timeout:   IsErrorTimeouts(o.Timeouts),
    189 	}
    190 }
    191 
    192 // MaxAttempts returns the maximum number of attempts that can be made for a
    193 // request before failing.
    194 func (s *Standard) MaxAttempts() int {
    195 	return s.options.MaxAttempts
    196 }
    197 
    198 // IsErrorRetryable returns if the error is can be retried or not. Should not
    199 // consider the number of attempts made.
    200 func (s *Standard) IsErrorRetryable(err error) bool {
    201 	return s.retryable.IsErrorRetryable(err).Bool()
    202 }
    203 
    204 // RetryDelay returns the delay to use before another request attempt is made.
    205 func (s *Standard) RetryDelay(attempt int, err error) (time.Duration, error) {
    206 	return s.backoff.BackoffDelay(attempt, err)
    207 }
    208 
    209 // GetAttemptToken returns the token to be released after then attempt completes.
    210 // The release token will add NoRetryIncrement to the RateLimiter token pool if
    211 // the attempt was successful. If the attempt failed, nothing will be done.
    212 func (s *Standard) GetAttemptToken(context.Context) (func(error) error, error) {
    213 	return s.GetInitialToken(), nil
    214 }
    215 
    216 // GetInitialToken returns a token for adding the NoRetryIncrement to the
    217 // RateLimiter token if the attempt completed successfully without error.
    218 //
    219 // InitialToken applies to result of the each attempt, including the first.
    220 // Whereas the RetryToken applies to the result of subsequent attempts.
    221 //
    222 // Deprecated: use GetAttemptToken instead.
    223 func (s *Standard) GetInitialToken() func(error) error {
    224 	return releaseToken(s.noRetryIncrement).release
    225 }
    226 
    227 func (s *Standard) noRetryIncrement() error {
    228 	return s.options.RateLimiter.AddTokens(s.options.NoRetryIncrement)
    229 }
    230 
    231 // GetRetryToken attempts to deduct the retry cost from the retry token pool.
    232 // Returning the token release function, or error.
    233 func (s *Standard) GetRetryToken(ctx context.Context, opErr error) (func(error) error, error) {
    234 	cost := s.options.RetryCost
    235 
    236 	if s.timeout.IsErrorTimeout(opErr).Bool() {
    237 		cost = s.options.RetryTimeoutCost
    238 	}
    239 
    240 	fn, err := s.options.RateLimiter.GetToken(ctx, cost)
    241 	if err != nil {
    242 		return nil, fmt.Errorf("failed to get rate limit token, %w", err)
    243 	}
    244 
    245 	return releaseToken(fn).release, nil
    246 }
    247 
    248 func nopRelease(error) error { return nil }
    249 
    250 type releaseToken func() error
    251 
    252 func (f releaseToken) release(err error) error {
    253 	if err != nil {
    254 		return nil
    255 	}
    256 
    257 	return f()
    258 }