diff options
Diffstat (limited to 'crawler')
| -rw-r--r-- | crawler/scrape.go | 25 |
1 files changed, 16 insertions, 9 deletions
diff --git a/crawler/scrape.go b/crawler/scrape.go index 6483548..246f333 100644 --- a/crawler/scrape.go +++ b/crawler/scrape.go @@ -28,17 +28,24 @@ func (app *App) Scrape(shop Shop, wait chan bool) { var W []Angebot var err error - W = app.ScrapeShop(shop) + // retry on error + for i := 0; i < 3; i++ { + W = app.ScrapeShop(shop) - W = sanitize_offer(W, shop) + W = sanitize_offer(W, shop) - err = app.save_offer(W) - if err != nil { - Warn(err, "Saving offers failed") - } - err = app.remove_expired(W, shop) - if err != nil { - Warn(err, "Removing expired offers failed") + if len(W) >= 1 { + break + } + + err = app.save_offer(W) + if err != nil { + Warn(err, "Saving offers failed") + } + err = app.remove_expired(W, shop) + if err != nil { + Warn(err, "Removing expired offers failed") + } } wait <- true |