Adds structured logging. (crawler)

1 files changed, 37 insertions, 7 deletions

diff --git a/crawler/shop_rumundco.go b/crawler/shop_rumundco.go
index cd67c0f..e1516ba 100644
--- a/crawler/shop_rumundco.go
+++ b/crawler/shop_rumundco.go
@@ -1,7 +1,6 @@
 package main
 
 import (
-	"log"
 	"regexp"
 	"strings"
 
@@ -33,7 +32,7 @@ func ScrapeRumundCo(shop Shop) []Angebot {
 
 		matched, err := regexp.MatchString("verfügbar", e.ChildText(".delivery-status"))
 		if err != nil {
-			log.Fatal(err)
+			Fatal(err, "Rum & Co: Verfügbar regex failed")
 		}
 		if !matched {
 			return
@@ -42,41 +41,72 @@ func ScrapeRumundCo(shop Shop) []Angebot {
 		W.Name = whisky_name
 		W.Url = whisky_url
 
+		r_abv, err := regexp.Compile("[0-9]+([,.][0-9])?( )*(%|([vV]ol))")
+		if err != nil {
+			Fatal(err, "Rum & Co: Abv regex failed")
+		}
+		abv_noisy := r_abv.FindString(whisky_name)
+
 		e.ForEach(".price_wrapper", func(i int, e *colly.HTMLElement) {
 			regular_price := e.ChildText("del.value")
 			if "" == regular_price {
+				PrintlnOffer(W, "Rum & Co: No regular price found")
 				return
 			}
 			W.Original_price, err = convert_price(regular_price)
 			if err != nil {
-				log.Fatal(err)
+				Fatal(err, "Rum & Co: Original price: Convert price failed")
 			}
 			W.Discounted_price, err = convert_price(e.ChildText(".price-value"))
 			if err != nil {
-				log.Fatal(err)
+				Fatal(err, "Rum & Co: Discounted price: Convert price failed")
 			}
 
 			e.ForEach(".base_price", func(i int, e *colly.HTMLElement) {
 				price_per_litre_noisy := e.ChildText(".value")
 				W.Base_price, err = sanitize_base_price(price_per_litre_noisy)
 				if err != nil {
-					log.Fatal(err)
+					Fatal(err, "Rum & Co: Base price: Sanitizing base price failed")
 				}
 			})
 
 		})
 
-		W.Image_url = "https://www.rumundco.de/" + e.ChildAttr("img", "src")
+		// Rum & Co uses pagespeed
+		image_url_noisy := e.ChildAttr("img", "src")
+
+		if strings.Contains(image_url_noisy, "pagespeed") {
+			r_pagespeed, err := regexp.Compile(`jpg(\.pagespeed.+)$`)
+			if err != nil {
+				Fatal(err, "Rum & Co: Pagespeed regexp failed")
+			}
+			image_url_noisy_slice := r_pagespeed.FindStringSubmatch(image_url_noisy)
+			if len(image_url_noisy_slice) < 2 {
+				PrintlnOffer(W, "Rum & Co: (Pagespeed) Image URL not found")
+				return
+			}
+			image_url_noisy = strings.Replace(image_url_noisy, image_url_noisy_slice[1], "", 1)
+		}
+
+		W.Image_url = "https://www.rumundco.de/" + image_url_noisy
 
 		e.Request.Visit(W.Url)
 
 		W.Volume = get_volume(e)
 
 		if W.Volume == 0 {
+			PrintlnOffer(W, "Rum & Co: No Volume found")
 			return
 		}
 
-		W.Abv = get_abv(e)
+		if "" == abv_noisy {
+			W.Abv = get_abv(e)
+		} else {
+			W.Abv, err = extract_abv(abv_noisy)
+			if err != nil {
+				Fatal(err, "Rum & Co: Base price: Extracting ABV failed")
+			}
+		}
 
 		W.Shop = shop.Id
 		W.Spirit_type = "Whisky"