Merge branch 'master' of /home/horus/app/fk_angebote

1 files changed, 9 insertions, 7 deletions

diff --git a/crawler/shop_whiskyworld.go b/crawler/shop_whiskyworld.go
index af97511..3f0874d 100644
--- a/crawler/shop_whiskyworld.go
+++ b/crawler/shop_whiskyworld.go
@@ -15,11 +15,7 @@ func (app *App) ScrapeWhiskyworld(shop Shop) []Angebot {
 
 	Whiskys := []Angebot{}
 
-	c := colly.NewCollector(
-		colly.UserAgent("friendly"),
-		colly.AllowedDomains("whiskyworld.de"),
-		colly.AllowedDomains("www.whiskyworld.de"),
-	)
+	c := app.customCollector([]string{"whiskyworld.de", "www.whiskyworld.de"})
 
 	c.OnHTML(".product-item", func(e *colly.HTMLElement) {
 		if !stringInSlice(e.Request.URL.String(), Shop_urls) {
@@ -106,7 +102,10 @@ func (app *App) ScrapeWhiskyworld(shop Shop) []Angebot {
 
 		})
 
-		W.Image_url = "https:" + e.ChildAttr("img", "data-src")
+		W.Image_url = e.ChildAttr("img", "data-src")
+		if !strings.HasPrefix(W.Image_url, "https:") {
+			W.Image_url = "https:" + W.Image_url
+		}
 
 		e.Request.Visit(W.Url)
 		W.Website = e.Request.Ctx.Get("website")
@@ -124,7 +123,10 @@ func (app *App) ScrapeWhiskyworld(shop Shop) []Angebot {
 	})
 
 	for _, url := range Shop_urls {
-		c.Visit(url)
+		err := c.Visit(url)
+		if err != nil {
+			Warn(nil, shop.Name+": "+err.Error())
+		}
 	}
 
 	return Whiskys