standard.go (7979B)
1 package retry 2 3 import ( 4 "context" 5 "fmt" 6 "time" 7 8 "github.com/aws/aws-sdk-go-v2/aws/ratelimit" 9 ) 10 11 // BackoffDelayer provides the interface for determining the delay to before 12 // another request attempt, that previously failed. 13 type BackoffDelayer interface { 14 BackoffDelay(attempt int, err error) (time.Duration, error) 15 } 16 17 // BackoffDelayerFunc provides a wrapper around a function to determine the 18 // backoff delay of an attempt retry. 19 type BackoffDelayerFunc func(int, error) (time.Duration, error) 20 21 // BackoffDelay returns the delay before attempt to retry a request. 22 func (fn BackoffDelayerFunc) BackoffDelay(attempt int, err error) (time.Duration, error) { 23 return fn(attempt, err) 24 } 25 26 const ( 27 // DefaultMaxAttempts is the maximum of attempts for an API request 28 DefaultMaxAttempts int = 3 29 30 // DefaultMaxBackoff is the maximum back off delay between attempts 31 DefaultMaxBackoff time.Duration = 20 * time.Second 32 ) 33 34 // Default retry token quota values. 35 const ( 36 DefaultRetryRateTokens uint = 500 37 DefaultRetryCost uint = 5 38 DefaultRetryTimeoutCost uint = 10 39 DefaultNoRetryIncrement uint = 1 40 ) 41 42 // DefaultRetryableHTTPStatusCodes is the default set of HTTP status codes the SDK 43 // should consider as retryable errors. 44 var DefaultRetryableHTTPStatusCodes = map[int]struct{}{ 45 500: {}, 46 502: {}, 47 503: {}, 48 504: {}, 49 } 50 51 // DefaultRetryableErrorCodes provides the set of API error codes that should 52 // be retried. 53 var DefaultRetryableErrorCodes = map[string]struct{}{ 54 "RequestTimeout": {}, 55 "RequestTimeoutException": {}, 56 } 57 58 // DefaultThrottleErrorCodes provides the set of API error codes that are 59 // considered throttle errors. 60 var DefaultThrottleErrorCodes = map[string]struct{}{ 61 "Throttling": {}, 62 "ThrottlingException": {}, 63 "ThrottledException": {}, 64 "RequestThrottledException": {}, 65 "TooManyRequestsException": {}, 66 "ProvisionedThroughputExceededException": {}, 67 "TransactionInProgressException": {}, 68 "RequestLimitExceeded": {}, 69 "BandwidthLimitExceeded": {}, 70 "LimitExceededException": {}, 71 "RequestThrottled": {}, 72 "SlowDown": {}, 73 "PriorRequestNotComplete": {}, 74 "EC2ThrottledException": {}, 75 } 76 77 // DefaultRetryables provides the set of retryable checks that are used by 78 // default. 79 var DefaultRetryables = []IsErrorRetryable{ 80 NoRetryCanceledError{}, 81 RetryableError{}, 82 RetryableConnectionError{}, 83 RetryableHTTPStatusCode{ 84 Codes: DefaultRetryableHTTPStatusCodes, 85 }, 86 RetryableErrorCode{ 87 Codes: DefaultRetryableErrorCodes, 88 }, 89 RetryableErrorCode{ 90 Codes: DefaultThrottleErrorCodes, 91 }, 92 } 93 94 // DefaultTimeouts provides the set of timeout checks that are used by default. 95 var DefaultTimeouts = []IsErrorTimeout{ 96 TimeouterError{}, 97 } 98 99 // StandardOptions provides the functional options for configuring the standard 100 // retryable, and delay behavior. 101 type StandardOptions struct { 102 // Maximum number of attempts that should be made. 103 MaxAttempts int 104 105 // MaxBackoff duration between retried attempts. 106 MaxBackoff time.Duration 107 108 // Provides the backoff strategy the retryer will use to determine the 109 // delay between retry attempts. 110 Backoff BackoffDelayer 111 112 // Set of strategies to determine if the attempt should be retried based on 113 // the error response received. 114 // 115 // It is safe to append to this list in NewStandard's functional options. 116 Retryables []IsErrorRetryable 117 118 // Set of strategies to determine if the attempt failed due to a timeout 119 // error. 120 // 121 // It is safe to append to this list in NewStandard's functional options. 122 Timeouts []IsErrorTimeout 123 124 // Provides the rate limiting strategy for rate limiting attempt retries 125 // across all attempts the retryer is being used with. 126 RateLimiter RateLimiter 127 128 // The cost to deduct from the RateLimiter's token bucket per retry. 129 RetryCost uint 130 131 // The cost to deduct from the RateLimiter's token bucket per retry caused 132 // by timeout error. 133 RetryTimeoutCost uint 134 135 // The cost to payback to the RateLimiter's token bucket for successful 136 // attempts. 137 NoRetryIncrement uint 138 } 139 140 // RateLimiter provides the interface for limiting the rate of attempt retries 141 // allowed by the retryer. 142 type RateLimiter interface { 143 GetToken(ctx context.Context, cost uint) (releaseToken func() error, err error) 144 AddTokens(uint) error 145 } 146 147 // Standard is the standard retry pattern for the SDK. It uses a set of 148 // retryable checks to determine of the failed attempt should be retried, and 149 // what retry delay should be used. 150 type Standard struct { 151 options StandardOptions 152 153 timeout IsErrorTimeout 154 retryable IsErrorRetryable 155 backoff BackoffDelayer 156 } 157 158 // NewStandard initializes a standard retry behavior with defaults that can be 159 // overridden via functional options. 160 func NewStandard(fnOpts ...func(*StandardOptions)) *Standard { 161 o := StandardOptions{ 162 MaxAttempts: DefaultMaxAttempts, 163 MaxBackoff: DefaultMaxBackoff, 164 Retryables: append([]IsErrorRetryable{}, DefaultRetryables...), 165 Timeouts: append([]IsErrorTimeout{}, DefaultTimeouts...), 166 167 RateLimiter: ratelimit.NewTokenRateLimit(DefaultRetryRateTokens), 168 RetryCost: DefaultRetryCost, 169 RetryTimeoutCost: DefaultRetryTimeoutCost, 170 NoRetryIncrement: DefaultNoRetryIncrement, 171 } 172 for _, fn := range fnOpts { 173 fn(&o) 174 } 175 if o.MaxAttempts <= 0 { 176 o.MaxAttempts = DefaultMaxAttempts 177 } 178 179 backoff := o.Backoff 180 if backoff == nil { 181 backoff = NewExponentialJitterBackoff(o.MaxBackoff) 182 } 183 184 return &Standard{ 185 options: o, 186 backoff: backoff, 187 retryable: IsErrorRetryables(o.Retryables), 188 timeout: IsErrorTimeouts(o.Timeouts), 189 } 190 } 191 192 // MaxAttempts returns the maximum number of attempts that can be made for a 193 // request before failing. 194 func (s *Standard) MaxAttempts() int { 195 return s.options.MaxAttempts 196 } 197 198 // IsErrorRetryable returns if the error is can be retried or not. Should not 199 // consider the number of attempts made. 200 func (s *Standard) IsErrorRetryable(err error) bool { 201 return s.retryable.IsErrorRetryable(err).Bool() 202 } 203 204 // RetryDelay returns the delay to use before another request attempt is made. 205 func (s *Standard) RetryDelay(attempt int, err error) (time.Duration, error) { 206 return s.backoff.BackoffDelay(attempt, err) 207 } 208 209 // GetAttemptToken returns the token to be released after then attempt completes. 210 // The release token will add NoRetryIncrement to the RateLimiter token pool if 211 // the attempt was successful. If the attempt failed, nothing will be done. 212 func (s *Standard) GetAttemptToken(context.Context) (func(error) error, error) { 213 return s.GetInitialToken(), nil 214 } 215 216 // GetInitialToken returns a token for adding the NoRetryIncrement to the 217 // RateLimiter token if the attempt completed successfully without error. 218 // 219 // InitialToken applies to result of the each attempt, including the first. 220 // Whereas the RetryToken applies to the result of subsequent attempts. 221 // 222 // Deprecated: use GetAttemptToken instead. 223 func (s *Standard) GetInitialToken() func(error) error { 224 return releaseToken(s.noRetryIncrement).release 225 } 226 227 func (s *Standard) noRetryIncrement() error { 228 return s.options.RateLimiter.AddTokens(s.options.NoRetryIncrement) 229 } 230 231 // GetRetryToken attempts to deduct the retry cost from the retry token pool. 232 // Returning the token release function, or error. 233 func (s *Standard) GetRetryToken(ctx context.Context, opErr error) (func(error) error, error) { 234 cost := s.options.RetryCost 235 236 if s.timeout.IsErrorTimeout(opErr).Bool() { 237 cost = s.options.RetryTimeoutCost 238 } 239 240 fn, err := s.options.RateLimiter.GetToken(ctx, cost) 241 if err != nil { 242 return nil, fmt.Errorf("failed to get rate limit token, %w", err) 243 } 244 245 return releaseToken(fn).release, nil 246 } 247 248 func nopRelease(error) error { return nil } 249 250 type releaseToken func() error 251 252 func (f releaseToken) release(err error) error { 253 if err != nil { 254 return nil 255 } 256 257 return f() 258 }