package main import ( "encoding/json" "fmt" "io/ioutil" "net/http" //"strings" //"github.com/gocolly/colly" log "github.com/sirupsen/logrus" ) func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot { Offers := []Angebot{} /** * Parse the API. */ API_URL := "https://es-api.drankdozijn.nl/sale-products?country=DE&language=de" API_URL_PRODUCT := "https://es-api.drankdozijn.nl/product?country=DE&language=de&page_template=artikel&alias=" IMAGE_URL := "https://res-2.cloudinary.com/boozeboodcdn/image/upload/e_trim:10/c_pad,g_south,h_400,w_280/c_limit,h_910,w_280/f_auto,q_auto:best/v1/" http_client := http.Client{} req, err := http.NewRequest(http.MethodGet, API_URL, nil) if err != nil { // TODO panic(err) } req.Header.Set("accept", "application/json") req.Header.Set("User-Agent", "like googlebot") api_resp, err := http_client.Do(req) if err != nil { // TODO panic(err) } api_body, err := ioutil.ReadAll(api_resp.Body) if err != nil { // TODO panic(err) } //log.Println("%v\n", string(api_body)) type api_products struct { Alias string Images []string } type api_product_group struct { Description string } type api_offer struct { Type string Price float64 SalePrice float64 SaleDescription string Group map[string]interface{} ProductGroup api_product_group Products []api_products } type Value struct { Alias string `json:"alias"` Description string `json:"description"` } type Feature struct { Alias string `json:"alias"` Description string `json:"description"` Id int `json:"id"` Value Value `json:"value"` ShownOnPage bool `json:"shownOnPage"` } type product_details struct { FormatedPrice string `json:"formatedPrice"` FormatedSalePrice string `json:"formatedSalePrice"` Description string `json:"description"` Features []Feature `json:"features` } var offers []api_offer //err = json.Unmarshal(api_body, &tmp_api_map) err = json.Unmarshal(api_body, &offers) if err != nil { // TODO log.Println("offers json unmarshal failed") log.Printf("%+v\n", string(api_body)) panic(err) } for _, api_data := range offers { W := Angebot{} W.Shop = shop.Id W.Name = api_data.SaleDescription tmp_spirit_type := api_data.ProductGroup.Description if "Bier" == tmp_spirit_type { W.Debug("Drankdozijn: skip offer because it's beer") continue } W.Spirit_type = detect_spirit_type(tmp_spirit_type) if api_data.Price == 0 { //log.Println("%v\n", api_data["price"]) //log.Println("%v\n", api_data) W.Debug("Drankdozijn: price is nil -> skip offer") continue } W.Original_price, err = convert_price(fmt.Sprintf("%.2f", api_data.Price)) if err != nil { // TODO panic(err) } W.Discounted_price, err = convert_price(fmt.Sprintf("%.2f", api_data.SalePrice)) if err != nil { // TODO panic(err) } if W.Discounted_price >= W.Original_price { W.Debug("Drankdozijn: Discounted price is not cheaper") continue } // Offer URL //tmp_offer_url_img_map := api_data.Products["products"].(map[string]interface{}) tmp_offer_product_map := api_data.Products // Check for bundle offer if len(tmp_offer_product_map) > 1 { W.Debug("Drankdozijn: Skip Offer because of bundle") continue } var alias string for _, v := range tmp_offer_product_map { W.Url = "https://drankdozijn.de/artikel/" + v.Alias //tmp_image_map := tmp_map["images"].([]interface{}) //W.Image_url = IMAGE_URL + tmp_image_map[0].(string) if len(v.Images) > 0 { W.Image_url = IMAGE_URL + v.Images[0] alias = v.Alias } else { W.Image_url = "" } } if W.Image_url == "" { W.Debug("Drankdozijn: No image") continue } req, err := http.NewRequest(http.MethodGet, API_URL_PRODUCT+alias, nil) if err != nil { // TODO panic(err) } req.Header.Set("accept", "application/json") req.Header.Set("User-Agent", "like googlebot") api_resp, err := http_client.Do(req) if err != nil { // TODO panic(err) } api_body, err := ioutil.ReadAll(api_resp.Body) if err != nil { // TODO panic(err) } var d12_product product_details err = json.Unmarshal(api_body, &d12_product) if err != nil { // TODO log.Println("product_details json unmarshal failed") log.Printf("%+v\n", string(api_body)) panic(err) } W.Base_price, err = convert_price(d12_product.FormatedPrice) if err != nil { // TODO log.Println("converting price from product_details failed") panic(err) } for _, v := range d12_product.Features { if v.Description == "Alkoholgehalt" { W.Abv, err = extract_abv(v.Value.Description) if err != nil { log.Println("extracting abv failed") } } else if v.Description == "Inhalt" { W.Volume, err = extract_volume(v.Value.Description) if err != nil { log.Println("extracting volume failed") } } else if v.Description == "Kategorie" { var tmp_type string if "Moussierend" == v.Value.Description { tmp_type = "Sekt" } else { tmp_type = detect_spirit_type(v.Value.Description) } if "Champagner" == tmp_type || "Sekt" == tmp_type { if tmp_type != W.Spirit_type { W.Trace("Spirit Type Changed: " + W.Spirit_type + " -> " + tmp_type) W.Spirit_type = tmp_type } W.Spirit_type = tmp_type } switch W.Spirit_type { case "Cognac": if tmp_type != W.Spirit_type { W.Trace("Spirit Type Changed: " + W.Spirit_type + " -> " + tmp_type) W.Spirit_type = tmp_type } W.Spirit_type = tmp_type case "Brandy": if tmp_type != W.Spirit_type { W.Trace("Spirit Type Changed: " + W.Spirit_type + " -> " + tmp_type) W.Spirit_type = tmp_type } W.Spirit_type = tmp_type case "Sherry": if tmp_type != W.Spirit_type { W.Trace("Spirit Type Changed: " + W.Spirit_type + " -> " + tmp_type) W.Spirit_type = tmp_type } W.Spirit_type = tmp_type case "Likör": retest_type := detect_spirit_type(tmp_type) if "Likör" != retest_type && "Verschiedenes" != retest_type { if tmp_type != W.Spirit_type { W.Trace("Spirit Type Changed: " + W.Spirit_type + " -> " + retest_type) W.Spirit_type = tmp_type } W.Spirit_type = detect_spirit_type(W.Name) } if "Tequila" == tmp_type { if tmp_type != W.Spirit_type { W.Trace("Spirit Type Changed: " + W.Spirit_type + " -> " + tmp_type) W.Spirit_type = tmp_type } W.Spirit_type = tmp_type } if "Mezcal" == tmp_type { if tmp_type != W.Spirit_type { W.Trace("Spirit Type Changed: " + W.Spirit_type + " -> " + tmp_type) W.Spirit_type = tmp_type } W.Spirit_type = tmp_type } if "Baijiu" == tmp_type { if tmp_type != W.Spirit_type { W.Trace("Spirit Type Changed: " + W.Spirit_type + " -> " + tmp_type) W.Spirit_type = tmp_type } W.Spirit_type = tmp_type } if "Absinth" == tmp_type { if tmp_type != W.Spirit_type { W.Trace("Spirit Type Changed: " + W.Spirit_type + " -> " + tmp_type) W.Spirit_type = tmp_type } W.Spirit_type = tmp_type } } } } //W.Debug("DEBUG OFFER") if W.Abv == 0 { W.Println("Drankdozijn: Abv is zero") continue } Offers = append(Offers, W) } return Offers }