summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authordev2026-06-25 21:18:12 +0200
committerdev2026-06-25 21:18:12 +0200
commit82520cd838db155f51aa1dbeac3373de225f38a0 (patch)
treeeb8087558e5a297227097e6b387f51875346d8c2
parente13cbe6a4fd1ebe3f2c3bfc86c54e8dd17c59624 (diff)
downloadhnimdbbot-82520cd838db155f51aa1dbeac3373de225f38a0.tar.gz
fix: add 429 retry with exponential backoff and increase rate limit delay
- Retry up to 5 times on HTTP 429 with 2s/4s/8s/16s backoff - Move inter-request delay before each request (was after) - Increase base delay from 1s to 2s between requests - Fix: only sleep after first request (skip delay on first call)
-rw-r--r--src/wikiarticle.go41
1 files changed, 32 insertions, 9 deletions
diff --git a/src/wikiarticle.go b/src/wikiarticle.go
index 61a6084..4ff72b6 100644
--- a/src/wikiarticle.go
+++ b/src/wikiarticle.go
@@ -87,14 +87,16 @@ func (a *App) fetchWikiArticlesData() error {
// Serial processing with 1 req/s rate limit
go func() {
- for _, item := range entries {
+ for i, item := range entries {
+ if i > 0 {
+ time.Sleep(2 * time.Second)
+ }
entry, err := a.queryWikiArticle(item.wikiArticle)
if err != nil {
- log.Printf("wiki error for %s (%s): %v", item.imdbID, item.wikiArticle, err)
+ log.Printf("wiki error for %s/%d (%s): %v", item.imdbID, i, item.wikiArticle, err)
continue
}
ch <- result{id: item.id, entry: entry}
- time.Sleep(1 * time.Second)
}
close(ch)
}()
@@ -128,17 +130,38 @@ func (a *App) queryWikiArticle(name string) (wikiArticleEntry, error) {
"name": {name},
}.Encode()
- resp, err := wikiArticleClient.Get(reqURL)
+ var resp *http.Response
+ var err error
+ for attempt := 0; attempt < 5; attempt++ {
+ if attempt > 0 {
+ backoff := 1 << attempt
+ log.Printf("retry %d/%d for %s after %ds backoff", attempt, 4, name, backoff)
+ time.Sleep(time.Duration(backoff) * time.Second)
+ }
+
+ resp, err = wikiArticleClient.Get(reqURL)
+ if err != nil {
+ continue
+ }
+
+ if resp.StatusCode == http.StatusTooManyRequests {
+ resp.Body.Close()
+ continue
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ body, _ := io.ReadAll(io.LimitReader(resp.Body, 2048))
+ resp.Body.Close()
+ return wikiArticleEntry{}, fmt.Errorf("HTTP %d: %s", resp.StatusCode, body)
+ }
+
+ break
+ }
if err != nil {
return wikiArticleEntry{}, fmt.Errorf("http get: %w", err)
}
defer resp.Body.Close()
- if resp.StatusCode != http.StatusOK {
- body, _ := io.ReadAll(io.LimitReader(resp.Body, 2048))
- return wikiArticleEntry{}, fmt.Errorf("HTTP %d: %s", resp.StatusCode, body)
- }
-
var articles []map[string]interface{}
if err := json.NewDecoder(resp.Body).Decode(&articles); err != nil {
return wikiArticleEntry{}, fmt.Errorf("json decode: %w", err)