diff options
Diffstat (limited to 'crawler/shop_whiskyde.go')
| -rw-r--r-- | crawler/shop_whiskyde.go | 11 |
1 files changed, 6 insertions, 5 deletions
diff --git a/crawler/shop_whiskyde.go b/crawler/shop_whiskyde.go index 0245c85..d3087ca 100644 --- a/crawler/shop_whiskyde.go +++ b/crawler/shop_whiskyde.go @@ -11,14 +11,12 @@ func (app *App) ScrapeWhiskyde(shop Shop) []Angebot { Whiskys := []Angebot{} - c := colly.NewCollector( - colly.AllowedDomains("whisky.de"), - colly.AllowedDomains("www.whisky.de"), - ) + c := app.customCollector([]string{"whisky.de", "www.whisky.de"}) c.OnHTML(".is-buyable", func(e *colly.HTMLElement) { if e.Request.URL.String() != Shop_url { + Debug(nil, "Whisky.de: Request url ("+e.Request.URL.String()+") is not shop url ("+Shop_url+").") return } @@ -120,7 +118,10 @@ func (app *App) ScrapeWhiskyde(shop Shop) []Angebot { e.Request.Ctx.Put("website", string(e.Response.Body)) }) - c.Visit(Shop_url) + err := c.Visit(Shop_url) + if err != nil { + Warn(nil, shop.Name+": "+err.Error()) + } return Whiskys } |