Implements retry on error. (crawler)

1 files changed, 16 insertions, 9 deletions

diff --git a/crawler/scrape.go b/crawler/scrape.go
index 6483548..246f333 100644
--- a/crawler/scrape.go
+++ b/crawler/scrape.go
@@ -28,17 +28,24 @@ func (app *App) Scrape(shop Shop, wait chan bool) {
 	var W []Angebot
 	var err error
 
-	W = app.ScrapeShop(shop)
+	// retry on error
+	for i := 0; i < 3; i++ {
+		W = app.ScrapeShop(shop)
 
-	W = sanitize_offer(W, shop)
+		W = sanitize_offer(W, shop)
 
-	err = app.save_offer(W)
-	if err != nil {
-		Warn(err, "Saving offers failed")
-	}
-	err = app.remove_expired(W, shop)
-	if err != nil {
-		Warn(err, "Removing expired offers failed")
+		if len(W) >= 1 {
+			break
+		}
+
+		err = app.save_offer(W)
+		if err != nil {
+			Warn(err, "Saving offers failed")
+		}
+		err = app.remove_expired(W, shop)
+		if err != nil {
+			Warn(err, "Removing expired offers failed")
+		}
 	}
 
 	wait <- true