diff options
Diffstat (limited to 'crawler/shop_bottleworld.go')
| -rw-r--r-- | crawler/shop_bottleworld.go | 15 |
1 files changed, 9 insertions, 6 deletions
diff --git a/crawler/shop_bottleworld.go b/crawler/shop_bottleworld.go index b92896d..8722211 100644 --- a/crawler/shop_bottleworld.go +++ b/crawler/shop_bottleworld.go @@ -12,10 +12,7 @@ func (app *App) ScrapeBottleWord(shop Shop) []Angebot { Shop_url := "https://www.bottleworld.de/aktuelle-sonderpreise/show/all" Whiskys := []Angebot{} - c := colly.NewCollector( - colly.AllowedDomains("bottleworld.de"), - colly.AllowedDomains("www.bottleworld.de"), - ) + c := app.customCollector([]string{"bottleworld.de", "www.bottleworld.de"}) c.OnHTML("li.item", func(e *colly.HTMLElement) { W := Angebot{} @@ -63,7 +60,10 @@ func (app *App) ScrapeBottleWord(shop Shop) []Angebot { W.Image_url = e.ChildAttr("img", "src") - e.Request.Visit(W.Url) + erro := e.Request.Visit(W.Url) + if erro != nil { + Warn(nil, W.Url+" "+erro.Error()) + } var ctx string W.Volume, ctx = get_volume(e) @@ -112,7 +112,10 @@ func (app *App) ScrapeBottleWord(shop Shop) []Angebot { e.Request.Ctx.Put("spirit_type", detect_spirit_type(text_noisy)) }) - c.Visit(Shop_url) + err := c.Visit(Shop_url) + if err != nil { + Warn(nil, shop.Name+": "+err.Error()) + } return Whiskys } |