diff options
Diffstat (limited to 'crawler/shop_whiskyworld.go')
| -rw-r--r-- | crawler/shop_whiskyworld.go | 16 |
1 files changed, 9 insertions, 7 deletions
diff --git a/crawler/shop_whiskyworld.go b/crawler/shop_whiskyworld.go index af97511..3f0874d 100644 --- a/crawler/shop_whiskyworld.go +++ b/crawler/shop_whiskyworld.go @@ -15,11 +15,7 @@ func (app *App) ScrapeWhiskyworld(shop Shop) []Angebot { Whiskys := []Angebot{} - c := colly.NewCollector( - colly.UserAgent("friendly"), - colly.AllowedDomains("whiskyworld.de"), - colly.AllowedDomains("www.whiskyworld.de"), - ) + c := app.customCollector([]string{"whiskyworld.de", "www.whiskyworld.de"}) c.OnHTML(".product-item", func(e *colly.HTMLElement) { if !stringInSlice(e.Request.URL.String(), Shop_urls) { @@ -106,7 +102,10 @@ func (app *App) ScrapeWhiskyworld(shop Shop) []Angebot { }) - W.Image_url = "https:" + e.ChildAttr("img", "data-src") + W.Image_url = e.ChildAttr("img", "data-src") + if !strings.HasPrefix(W.Image_url, "https:") { + W.Image_url = "https:" + W.Image_url + } e.Request.Visit(W.Url) W.Website = e.Request.Ctx.Get("website") @@ -124,7 +123,10 @@ func (app *App) ScrapeWhiskyworld(shop Shop) []Angebot { }) for _, url := range Shop_urls { - c.Visit(url) + err := c.Visit(url) + if err != nil { + Warn(nil, shop.Name+": "+err.Error()) + } } return Whiskys |