From e63a7541a38c25dc8a78e7f838e61c7318c4775f Mon Sep 17 00:00:00 2001
From: horus
Date: Mon, 19 Feb 2018 17:53:39 +0100
Subject: Implements retry on error. (crawler)

---
 crawler/scrape.go | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

(limited to 'crawler/scrape.go')

diff --git a/crawler/scrape.go b/crawler/scrape.go
index 6483548..246f333 100644
--- a/crawler/scrape.go
+++ b/crawler/scrape.go
@@ -28,17 +28,24 @@ func (app *App) Scrape(shop Shop, wait chan bool) {
 	var W []Angebot
 	var err error
 
-	W = app.ScrapeShop(shop)
+	// retry on error
+	for i := 0; i < 3; i++ {
+		W = app.ScrapeShop(shop)
 
-	W = sanitize_offer(W, shop)
+		W = sanitize_offer(W, shop)
 
-	err = app.save_offer(W)
-	if err != nil {
-		Warn(err, "Saving offers failed")
-	}
-	err = app.remove_expired(W, shop)
-	if err != nil {
-		Warn(err, "Removing expired offers failed")
+		if len(W) >= 1 {
+			break
+		}
+
+		err = app.save_offer(W)
+		if err != nil {
+			Warn(err, "Saving offers failed")
+		}
+		err = app.remove_expired(W, shop)
+		if err != nil {
+			Warn(err, "Removing expired offers failed")
+		}
 	}
 
 	wait <- true
-- 
cgit v1.2.3