package main import ( "regexp" "github.com/gocolly/colly" ) func (app *App) ScrapeMCWhisky(shop Shop) []Angebot { Shop_url := "https://www.mcwhisky.com/whisky/whisky-sonderangebote.html" Whiskys := []Angebot{} c := app.customCollector([]string{"mcwhisky.com", "www.mcwhisky.com"}) c.OnHTML("li.item", func(e *colly.HTMLElement) { if e.Request.URL.String() != Shop_url { return } W := Angebot{} W.Shop = shop.Id W.Spirit_type = "Whisky" whisky_name := e.ChildAttr("a", "title") whisky_url := e.ChildAttr("a", "href") W.Name = whisky_name W.Url = whisky_url var err error e.ForEach(".price-box", func(i int, e *colly.HTMLElement) { e.ForEach(".old-price", func(i int, e *colly.HTMLElement) { W.Original_price, err = convert_price(e.ChildText(".price")) if err != nil { W.error_msg = err.Error() W.error_ctx = e.ChildText(".price") PrintlnOffer(W, "MC Whisky: Converting original price failed") return } }) e.ForEach(".special-price", func(i int, e *colly.HTMLElement) { W.Discounted_price, err = convert_price(e.ChildText(".price")) if err != nil { W.error_msg = err.Error() W.error_ctx = e.ChildText(".price") PrintlnOffer(W, "MC Whisky: Converting discounted price failed") return } }) }) price_per_litre_noisy := e.ChildText(".price-box-extended-info-ppl") W.Base_price, err = sanitize_base_price(price_per_litre_noisy) if err != nil { W.error_msg = err.Error() W.error_ctx = price_per_litre_noisy PrintlnOffer(W, "MC Whisky: Sanitizing base price failed") return } W.Image_url = e.ChildAttr("img", "src") e.Request.Visit(W.Url) volume_failed := e.Request.Ctx.Get("volume_failed") if volume_failed != "" { W.error_msg = "MC Whisky: Volume failed" W.error_ctx = volume_failed PrintlnOffer(W, "MC Whisky: Volume failed") return } abv_failed := e.Request.Ctx.Get("abv_failed") if volume_failed != "" { W.error_msg = "MC Whisky: Abv failed" W.error_ctx = abv_failed PrintlnOffer(W, "MC Whisky: Abv failed") return } var ctx string W.Volume, ctx = get_volume(e) if W.Volume == 0 { W.error_msg = "MC Whisky: Volume is zero" W.error_ctx = ctx PrintlnOffer(W, "MC Whisky: Volume is zero") return } W.Abv, ctx = get_abv(e) if W.Abv == 0 { W.error_msg = "MC Whisky: Abv is zero" W.error_ctx = ctx PrintlnOffer(W, "MC Whisky: Abv is zero") return } W.Website = e.Request.Ctx.Get("website") Whiskys = append(Whiskys, W) }) c.OnHTML(".products-attributes-alcohol", func(e *colly.HTMLElement) { text_noisy := e.Text r_abv, err := regexp.Compile(`[0-9]+([,.][0-9]+)?( )?%`) if err != nil { Fatal(err, "MC Whisky: ABV regex failed") } abv := r_abv.FindString(text_noisy) if abv == "" { e.Request.Ctx.Put("abv_fail", text_noisy) return } e.Request.Ctx.Put("abv", abv) r_volume, err := regexp.Compile(`[0-9]+([,.][0-9]+)?( )?Liter$`) if err != nil { Fatal(err, "MC Whisky: Volume regex failed") } volume := r_volume.FindString(text_noisy) if volume == "" { e.Request.Ctx.Put("volume_fail", text_noisy) return } e.Request.Ctx.Put("volume", volume) e.Request.Ctx.Put("website", string(e.Response.Body)) }) err := c.Visit(Shop_url) if err != nil { Warn(nil, shop.Name+": "+err.Error()) } return Whiskys }