Introduces config for user agent, robots.txt and crawler delay. (crawler)

1 files changed, 5 insertions, 2 deletions

diff --git a/crawler/shop_whiskyworld.go b/crawler/shop_whiskyworld.go
index 7b57d37..3f0874d 100644
--- a/crawler/shop_whiskyworld.go
+++ b/crawler/shop_whiskyworld.go
@@ -15,7 +15,7 @@ func (app *App) ScrapeWhiskyworld(shop Shop) []Angebot {
 
 	Whiskys := []Angebot{}
 
-	c := customCollector([]string{"whiskyworld.de", "www.whiskyworld.de"})
+	c := app.customCollector([]string{"whiskyworld.de", "www.whiskyworld.de"})
 
 	c.OnHTML(".product-item", func(e *colly.HTMLElement) {
 		if !stringInSlice(e.Request.URL.String(), Shop_urls) {
@@ -123,7 +123,10 @@ func (app *App) ScrapeWhiskyworld(shop Shop) []Angebot {
 	})
 
 	for _, url := range Shop_urls {
-		c.Visit(url)
+		err := c.Visit(url)
+		if err != nil {
+			Warn(nil, shop.Name+": "+err.Error())
+		}
 	}
 
 	return Whiskys