package main import ( "encoding/json" "io/ioutil" "net/http" "strings" log "github.com/Sirupsen/logrus" "github.com/gocolly/colly" ) func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot { Offers := []Angebot{} /** * Parse the API. */ API_URL := "https://api.drankdozijn.nl/sale-products?country=DE&language=de" IMAGE_URL := "https://res-2.cloudinary.com/boozeboodcdn/image/upload/e_trim:10/c_pad,g_south,h_400,w_280/c_limit,h_910,w_280/f_auto,q_auto:best/v1/" http_client := http.Client{} req, err := http.NewRequest(http.MethodGet, API_URL, nil) if err != nil { // TODO panic(err) } req.Header.Set("accept", "application/json") req.Header.Set("User-Agent", "") api_resp, err := http_client.Do(req) if err != nil { // TODO panic(err) } api_body, err := ioutil.ReadAll(api_resp.Body) if err != nil { // TODO panic(err) } var tmp_api_map map[string]interface{} err = json.Unmarshal(api_body, &tmp_api_map) if err != nil { // TODO log.Println("json unmarshal failed") panic(err) } for _, value := range tmp_api_map { api_data := value.(map[string]interface{}) if api_data["type"] != "offer" { continue } W := Angebot{} W.Shop = shop.Id W.Name = api_data["saleDescription"].(string) tmp_desc := api_data["group"].(map[string]interface{}) tmp_spirit_type := tmp_desc["description"].(string) if "Bier" == tmp_spirit_type { DebugOffer(W, "Drankdozijn: skip offer because it's beer") continue } W.Spirit_type = detect_spirit_type(tmp_desc["description"].(string)) if v, _ := api_data["price"]; v == nil { //DebugOffer(W, "Drankdozijn: Skip Offer") DebugOffer(W, "Drankdozijn: price is nil -> skip offer") continue } W.Original_price, err = convert_price(api_data["price"].(string)) if err != nil { // TODO panic(err) } W.Discounted_price, err = convert_price(api_data["salePrice"].(string)) if err != nil { // TODO panic(err) } if W.Discounted_price >= W.Original_price { DebugOffer(W, "Drankdozijn: Discounted price is not cheaper") continue } // Offer URL tmp_offer_url_img_map := api_data["products"].(map[string]interface{}) // Check for bundle offer if len(tmp_offer_url_img_map) > 1 { DebugOffer(W, "Drankdozijn: Skip Offer because of bundle") continue } for _, v := range tmp_offer_url_img_map { tmp_map := v.(map[string]interface{}) W.Url = "https://drankdozijn.de/artikel/" + (tmp_map["alias"]).(string) tmp_image_map := tmp_map["images"].([]interface{}) W.Image_url = IMAGE_URL + tmp_image_map[0].(string) } c := app.customCollector([]string{"drankdozijn.de", "drankdozijn.nl"}) c.OnHTML(".product_top", func(e *colly.HTMLElement) { if strings.Contains(W.Name, "+ gratis") || strings.Contains(W.Name, "& gratis") { DebugOffer(W, "Drankdozijn: Skip Offer because it contains gratis ware") return } e.Request.Visit(W.Url) }) c.OnHTML(".main_price", func(e *colly.HTMLElement) { W.Base_price, err = convert_price(e.ChildText(".price_l")) if err != nil { W.error_msg = err.Error() W.error_ctx = e.ChildText(".price_l") PrintlnOffer(W, "Drankdozijn: Converting base price failed") return } }) c.OnHTML(".row .main_description", func(e *colly.HTMLElement) { prev := "" count := 0 e.ForEach(".col-xs-6", func(i int, e *colly.HTMLElement) { if count%2 == 0 { prev = e.Text } else { switch strings.TrimSpace(prev) { case "Inhalt", "Inhoud": W.Volume, err = extract_volume(e.Text) if W.Volume == 0 { W.error_msg = e.Text W.error_ctx = err.Error() PrintlnOffer(W, "Drankdozijn: Volume is zero, returning") return } case "Alkoholgehalt", "Alcoholpercentage": W.Abv, err = extract_abv(e.Text) if W.Abv == 0 { W.error_msg = "Drankdozijn: Abv is zero" W.error_ctx = err.Error() PrintlnOffer(W, "Drankdozijn: abv is zero") return } case "Kategorie", "Categorie": tmp_type := e.Text tmp_type = detect_spirit_type(tmp_type) if "Champagner" == tmp_type { if tmp_type != W.Spirit_type { DebugOffer(W, "Spirit Type Changed: "+W.Spirit_type+" -> "+tmp_type) W.Spirit_type = tmp_type } W.Spirit_type = tmp_type } switch W.Spirit_type { case "Cognac": if tmp_type != W.Spirit_type { DebugOffer(W, "Spirit Type Changed: "+W.Spirit_type+" -> "+tmp_type) W.Spirit_type = tmp_type } W.Spirit_type = tmp_type case "Brandy": if tmp_type != W.Spirit_type { DebugOffer(W, "Spirit Type Changed: "+W.Spirit_type+" -> "+tmp_type) W.Spirit_type = tmp_type } W.Spirit_type = tmp_type case "Sherry": if tmp_type != W.Spirit_type { DebugOffer(W, "Spirit Type Changed: "+W.Spirit_type+" -> "+tmp_type) W.Spirit_type = tmp_type } W.Spirit_type = tmp_type case "Likör": retest_type := detect_spirit_type(W.Name) if "Likör" != retest_type && "Verschiedenes" != retest_type { if tmp_type != W.Spirit_type { DebugOffer(W, "Spirit Type Changed: "+W.Spirit_type+" -> "+retest_type) W.Spirit_type = tmp_type } W.Spirit_type = detect_spirit_type(W.Name) } if "Tequila" == tmp_type { if tmp_type != W.Spirit_type { DebugOffer(W, "Spirit Type Changed: "+W.Spirit_type+" -> "+tmp_type) W.Spirit_type = tmp_type } W.Spirit_type = tmp_type } if "Mezcal" == tmp_type { if tmp_type != W.Spirit_type { DebugOffer(W, "Spirit Type Changed: "+W.Spirit_type+" -> "+tmp_type) W.Spirit_type = tmp_type } W.Spirit_type = tmp_type } if "Baijiu" == tmp_type { if tmp_type != W.Spirit_type { DebugOffer(W, "Spirit Type Changed: "+W.Spirit_type+" -> "+tmp_type) W.Spirit_type = tmp_type } W.Spirit_type = tmp_type } } } prev = "" } count++ }) }) c.OnHTML("body", func(e *colly.HTMLElement) { W.Website = string(e.Response.Body) }) err = c.Visit(W.Url) if err != nil { Warn(nil, shop.Name+": Error (Visit): "+err.Error()) } else { //log.Println("Visit " + W.Url) } //DebugOffer(W, "DEBUG OFFER") Offers = append(Offers, W) } return Offers }