Introduces config for user agent, robots.txt and crawler delay. (crawler)

1 files changed, 5 insertions, 2 deletions

diff --git a/crawler/shop_rumundco.go b/crawler/shop_rumundco.go
index 4b72c08..45069c2 100644
--- a/crawler/shop_rumundco.go
+++ b/crawler/shop_rumundco.go
@@ -14,7 +14,7 @@ func (app *App) ScrapeRumundCo(shop Shop) []Angebot {
 
 	Whiskys := []Angebot{}
 
-	c := customCollector([]string{"rumundco.de", "www.rumundco.de"})
+	c := app.customCollector([]string{"rumundco.de", "www.rumundco.de"})
 
 	c.OnHTML(".product-teaser", func(e *colly.HTMLElement) {
 
@@ -155,7 +155,10 @@ func (app *App) ScrapeRumundCo(shop Shop) []Angebot {
 		e.Request.Ctx.Put("website", string(e.Response.Body))
 	})
 
-	c.Visit(Shop_url)
+	err := c.Visit(Shop_url)
+	if err != nil {
+		Warn(nil, shop.Name+": "+err.Error())
+	}
 
 	return Whiskys
 }