Crawler extracts volume, price per litre and abv. (MC Whisky, Rum & Co, Whic)

1 files changed, 45 insertions, 1 deletions

diff --git a/crawler/shop_rumundco.go b/crawler/shop_rumundco.go
index d62250e..ef98e90 100644
--- a/crawler/shop_rumundco.go
+++ b/crawler/shop_rumundco.go
@@ -10,6 +10,9 @@ import (
 )
 
 func ScrapeRumundCo(shop Shop) []Angebot {
+
+	Shop_url := "https://www.rumundco.de/navi.php?q=4&kf=29&kk-suesse-von=0&kk-suesse-bis=100&kk-milde-von=0&kk-milde-bis=100&kk-wuerze-von=0&kk-wuerze-bis=100&kk-frucht-von=0&kk-frucht-bis=100&kk-torf-von=0&kk-torf-bis=100&hf=0&af=90&Sortierung=11&a=350"
+
 	Whiskys := []Angebot{}
 
 	c := colly.NewCollector(
@@ -18,6 +21,11 @@ func ScrapeRumundCo(shop Shop) []Angebot {
 	)
 
 	c.OnHTML(".product-teaser", func(e *colly.HTMLElement) {
+
+		if e.Request.URL.String() != Shop_url {
+			return
+		}
+
 		W := Angebot{}
 
 		whisky_name := strings.TrimPrefix(e.ChildAttr("img", "alt"), "Restposten: ")
@@ -47,16 +55,52 @@ func ScrapeRumundCo(shop Shop) []Angebot {
 			if err != nil {
 				log.Fatal(err)
 			}
+
+			e.ForEach(".base_price", func(i int, e *colly.HTMLElement) {
+				price_per_litre_noisy := e.ChildText(".value")
+				W.Base_price, err = sanitize_base_price(price_per_litre_noisy)
+				if err != nil {
+					log.Fatal(err)
+				}
+			})
+
 		})
+
 		W.Image_url = "https://www.rumundco.de/" + e.ChildAttr("img", "src")
 
+		e.Request.Visit(W.Url)
+
+		W.Volume = get_volume(e)
+
+		if W.Volume == 0 {
+			return
+		}
+
+		W.Abv = get_abv(e)
+
 		W.Shop = shop.Id
 		W.Spirit_type = "Whisky"
 
 		Whiskys = append(Whiskys, W)
 	})
 
-	c.Visit("https://www.rumundco.de/navi.php?q=4&kf=29&kk-suesse-von=0&kk-suesse-bis=100&kk-milde-von=0&kk-milde-bis=100&kk-wuerze-von=0&kk-wuerze-bis=100&kk-frucht-von=0&kk-frucht-bis=100&kk-torf-von=0&kk-torf-bis=100&hf=0&af=90&Sortierung=11&a=350")
+	c.OnHTML("#table-collapse .product-attributes table", func(e *colly.HTMLElement) {
+
+		e.ForEach("tr", func(i int, e *colly.HTMLElement) {
+			text_noisy := e.ChildText("th")
+
+			//log.Println("Visiting (" + e.Request.URL.String() + "). Found: " + text_noisy + " END")
+
+			if strings.Contains(text_noisy, "Genauer Inhalt:") {
+				//log.Println("Visiting (" + e.Request.URL.String() + "). Found (V): " + e.ChildText("td") + " END")
+				e.Request.Ctx.Put("volume", e.ChildText("td"))
+			} else if strings.Contains(text_noisy, "Alkoholgehalt in %:") {
+				e.Request.Ctx.Put("abv", e.ChildText("a"))
+			}
+		})
+	})
+
+	c.Visit(Shop_url)
 
 	return Whiskys
 }