package main import ( "strings" "github.com/gocolly/colly" ) func (app *App) ScrapeWhiskyde(shop Shop) []Angebot { Shop_url := "https://www.whisky.de/shop/Angebote/Sonderangebote/" Whiskys := []Angebot{} c := app.customCollector([]string{"whisky.de", "www.whisky.de"}) c.OnHTML(".is-buyable", func(e *colly.HTMLElement) { if e.Request.URL.String() != Shop_url { Debug(nil, "Whisky.de: Request url ("+e.Request.URL.String()+") is not shop url ("+Shop_url+").") return } W := Angebot{} W.Shop = shop.Id W.Spirit_type = "Whisky" whisky_name := e.ChildAttr("a", "title") W.Name = whisky_name whisky_url := strings.Replace(e.ChildAttr("a", "href"), "?&searchorigin=2", "", 1) W.Url = whisky_url var err error e.ForEach(".article-price-original", func(i int, e *colly.HTMLElement) { W.Original_price, err = convert_price(e.ChildText("del")) if err != nil { W.error_msg = err.Error() W.error_ctx = e.ChildText("del") W.Println("Whisky.de: Converting original price failed") return } }) e.ForEach(".article-price", func(i int, e *colly.HTMLElement) { W.Discounted_price, err = convert_price(e.ChildText(".article-price-default")) if err != nil { W.error_msg = err.Error() W.error_ctx = e.ChildText(".article-price-default") W.Println("Whisky.de: Converting discounted price failed") return } }) e.ForEach(".article-thumbnail", func(i int, e *colly.HTMLElement) { W.Image_url = e.ChildAttr("img", "data-src") }) e.ForEach(".article-price-prefix", func(i int, e *colly.HTMLElement) { //W.Valid_until = e.ChildText(".article-price-special") }) text_noisy := e.ChildText(".article-amount") if !strings.Contains(text_noisy, "Liter") { W.error_ctx = text_noisy W.error_msg = "Whisky.de: String 'Liter' not found." W.Println("Whisky.de: String 'Liter' not found.") return } abv_noisy := strings.TrimSpace(strings.SplitAfter(text_noisy, "Liter")[1]) W.Volume, err = extract_volume(text_noisy) if err != nil { W.error_msg = err.Error() W.error_ctx = text_noisy W.Println("Whisky.de: Extracting volume failed") return } W.Abv, err = extract_abv(abv_noisy) if err != nil { W.error_msg = err.Error() W.error_ctx = abv_noisy W.Println("Whisky.de: Extracting abv failed") return } if W.Volume == 0 { W.error_msg = "Whisky.de: Volume is zero" W.error_ctx = text_noisy W.Println("Whisky.de: Volume is zero") return } if W.Abv == 0 { W.error_msg = "Whisky.de: Abv is zero" W.error_ctx = abv_noisy W.Println("Whisky.de: Abv is zero") return } W.Base_price, err = convert_price(e.ChildText(".article-unitprice-default")) if err != nil { W.error_msg = err.Error() W.error_ctx = e.ChildText(".article-unitprice-default") W.Println("Whisky.de: Converting base price failed") return } e.Request.Visit(W.Url) W.Website = e.Request.Ctx.Get("website") Whiskys = append(Whiskys, W) }) c.OnHTML("body", func(e *colly.HTMLElement) { if e.Request.URL.String() == Shop_url { return } e.Request.Ctx.Put("website", string(e.Response.Body)) }) err := c.Visit(Shop_url) if err != nil { Warn(nil, shop.Name+": "+err.Error()) } return Whiskys }