diff options
| author | dev | 2026-06-25 21:18:12 +0200 |
|---|---|---|
| committer | dev | 2026-06-25 21:18:12 +0200 |
| commit | 82520cd838db155f51aa1dbeac3373de225f38a0 (patch) | |
| tree | eb8087558e5a297227097e6b387f51875346d8c2 | |
| parent | e13cbe6a4fd1ebe3f2c3bfc86c54e8dd17c59624 (diff) | |
| download | hnimdbbot-82520cd838db155f51aa1dbeac3373de225f38a0.tar.gz | |
fix: add 429 retry with exponential backoff and increase rate limit delay
- Retry up to 5 times on HTTP 429 with 2s/4s/8s/16s backoff
- Move inter-request delay before each request (was after)
- Increase base delay from 1s to 2s between requests
- Fix: only sleep after first request (skip delay on first call)
| -rw-r--r-- | src/wikiarticle.go | 41 |
1 files changed, 32 insertions, 9 deletions
diff --git a/src/wikiarticle.go b/src/wikiarticle.go index 61a6084..4ff72b6 100644 --- a/src/wikiarticle.go +++ b/src/wikiarticle.go @@ -87,14 +87,16 @@ func (a *App) fetchWikiArticlesData() error { // Serial processing with 1 req/s rate limit go func() { - for _, item := range entries { + for i, item := range entries { + if i > 0 { + time.Sleep(2 * time.Second) + } entry, err := a.queryWikiArticle(item.wikiArticle) if err != nil { - log.Printf("wiki error for %s (%s): %v", item.imdbID, item.wikiArticle, err) + log.Printf("wiki error for %s/%d (%s): %v", item.imdbID, i, item.wikiArticle, err) continue } ch <- result{id: item.id, entry: entry} - time.Sleep(1 * time.Second) } close(ch) }() @@ -128,17 +130,38 @@ func (a *App) queryWikiArticle(name string) (wikiArticleEntry, error) { "name": {name}, }.Encode() - resp, err := wikiArticleClient.Get(reqURL) + var resp *http.Response + var err error + for attempt := 0; attempt < 5; attempt++ { + if attempt > 0 { + backoff := 1 << attempt + log.Printf("retry %d/%d for %s after %ds backoff", attempt, 4, name, backoff) + time.Sleep(time.Duration(backoff) * time.Second) + } + + resp, err = wikiArticleClient.Get(reqURL) + if err != nil { + continue + } + + if resp.StatusCode == http.StatusTooManyRequests { + resp.Body.Close() + continue + } + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 2048)) + resp.Body.Close() + return wikiArticleEntry{}, fmt.Errorf("HTTP %d: %s", resp.StatusCode, body) + } + + break + } if err != nil { return wikiArticleEntry{}, fmt.Errorf("http get: %w", err) } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(io.LimitReader(resp.Body, 2048)) - return wikiArticleEntry{}, fmt.Errorf("HTTP %d: %s", resp.StatusCode, body) - } - var articles []map[string]interface{} if err := json.NewDecoder(resp.Body).Decode(&articles); err != nil { return wikiArticleEntry{}, fmt.Errorf("json decode: %w", err) |
