diff options
| -rw-r--r-- | crawler/scrape.go | 4 | ||||
| -rw-r--r-- | crawler/shop_whiskysitenl.go | 24 |
2 files changed, 19 insertions, 9 deletions
diff --git a/crawler/scrape.go b/crawler/scrape.go index ae63e5c..6483548 100644 --- a/crawler/scrape.go +++ b/crawler/scrape.go @@ -57,8 +57,8 @@ func (app *App) ScrapeShop(shop Shop) []Angebot { return app.ScrapeWhic(shop) case "Whisky.de": return app.ScrapeWhiskyde(shop) - //case "Whiskysite.nl": - // return app.ScrapeWhiskysitenl(shop) + case "Whiskysite.nl": + return app.ScrapeWhiskysitenl(shop) case "Whisky World": return app.ScrapeWhiskyworld(shop) case "Whiskyzone": diff --git a/crawler/shop_whiskysitenl.go b/crawler/shop_whiskysitenl.go index fb2940d..d69ec68 100644 --- a/crawler/shop_whiskysitenl.go +++ b/crawler/shop_whiskysitenl.go @@ -19,8 +19,15 @@ func (app *App) ScrapeWhiskysitenl(shop Shop) []Angebot { c.OnHTML(".product-block", func(e *colly.HTMLElement) { + if e.Request.URL.String() != Shop_url { + return + } + W := Angebot{} + W.Shop = shop.Id + W.Spirit_type = "Whisky" + whisky_name := e.ChildAttr("img", "alt") whisky_url := e.ChildAttr("a", "href") @@ -52,9 +59,12 @@ func (app *App) ScrapeWhiskysitenl(shop Shop) []Angebot { W.Image_url = e.ChildAttr("img", "src") - if e.Request.Ctx.Get("volume_failed") != "" { + e.Request.Visit(W.Url) + + volume_failed := e.Request.Ctx.Get("volume_failed") + if volume_failed != "" { W.error_msg = "Whiskysite.nl: Extracting volume via Liter-Regex failed" - W.error_ctx = e.Request.Ctx.Get("volume_failed") + W.error_ctx = volume_failed WarnOffer(W, "Whiskysite.nl: Extracting volume via Liter-Regex failed") return } @@ -73,7 +83,7 @@ func (app *App) ScrapeWhiskysitenl(shop Shop) []Angebot { WarnOffer(W, "Whiskysite.nl: Extracting volume failed") return } - W.Abv, ctx = get_volume(e) + W.Abv, ctx = get_abv(e) if W.Abv == 0 { W.error_msg = "Whiskysite.nl: Extracting abv failed" W.error_ctx = ctx @@ -84,9 +94,6 @@ func (app *App) ScrapeWhiskysitenl(shop Shop) []Angebot { // calculate base price, volume is never zero W.Base_price = int(RoundToEven(float64(W.Discounted_price) / float64(W.Volume))) - W.Shop = shop.Id - W.Spirit_type = "Whisky" - Whiskys = append(Whiskys, W) }) @@ -112,7 +119,10 @@ func (app *App) ScrapeWhiskysitenl(shop Shop) []Angebot { e.Request.Ctx.Put("volume_failed", text_noisy) return } - e.Request.Ctx.Put("volume", r_number.FindString(litre_noisy)) + /* + * it's important to add "Liter", because it's required for get_volume() + */ + e.Request.Ctx.Put("volume", r_number.FindString(litre_noisy)+" Liter") r_abv, err := regexp.Compile("[0-9]+([.,][0-9]+)?( )*%") if err != nil { |