Adds structured logging. (crawler)

17 files changed, 196 insertions, 64 deletions

diff --git a/crawler/convert_price.go b/crawler/convert_price.go
index f1c88b4..b28639f 100644
--- a/crawler/convert_price.go
+++ b/crawler/convert_price.go
@@ -2,7 +2,6 @@ package main
 
 import (
 	"errors"
-	"log"
 	"regexp"
 	"strconv"
 	"strings"
@@ -98,7 +97,7 @@ func convert_price(price string) (int, error) {
 	*/
 	price_int, err := strconv.Atoi(price)
 	if err != nil {
-		log.Println(price)
+		Println(err, price)
 		return 0, err
 	}
 
diff --git a/crawler/database.go b/crawler/database.go
index 1f38ca1..2608c59 100644
--- a/crawler/database.go
+++ b/crawler/database.go
@@ -2,8 +2,9 @@ package main
 
 import (
 	"database/sql"
-	"log"
 	"strings"
+
+	log "github.com/Sirupsen/logrus"
 )
 
 func (app *App) createTables() error {
@@ -59,8 +60,10 @@ func (app *App) createTables() error {
 
 		view_query := `CREATE OR REPLACE VIEW ` + v + `_view AS
 		SELECT
-			angebot.id, angebot.name, angebot.abv, angebot.volume, angebot.url as long_url, angebot.short_url as url,original_price/100 as original_price, discounted_price/100 as discounted_price, angebot.base_price/100 as base_price, image_url,
-			shop.name as shop, shop.short_url as shop_url, shop.shipping_costs/100 as shipping_costs, shop.free_shipping, ROUND(100-((discounted_price/original_price)*100)) AS procent, spirit_type, created_at
+			angebot.id, angebot.name, angebot.abv, angebot.volume, angebot.url as long_url, angebot.short_url as url, spirit_type,
+			original_price/100 as original_price, discounted_price/100 as discounted_price, angebot.base_price/100 as base_price, image_url,
+			shop.name as shop, shop.id as shop_id, shop.short_url as shop_url, shop.shipping_costs/100 as shipping_costs, shop.free_shipping,
+			ROUND(100-((discounted_price/original_price)*100)) AS procent, created_at
 		FROM angebot
 		JOIN shop ON angebot.shop = shop.id
 		WHERE 
@@ -80,6 +83,7 @@ func (app *App) save_offer(W []Angebot) error {
 
 	stmt, err := app.DB.Prepare(query)
 	if err != nil {
+		Debug(err, "Save Offer: Preparing query failed")
 		return err
 	}
 	defer stmt.Close()
@@ -92,7 +96,7 @@ func (app *App) save_offer(W []Angebot) error {
 			continue
 		}
 
-		err := app.DB.QueryRow("SELECT 1 FROM angebot WHERE shop = ? AND name = ? AND url = ? AND original_price = ? AND discounted_price = ? AND spirit_type = ?", o.Shop, o.Name, o.Url, o.Original_price, o.Discounted_price, o.Spirit_type).Scan(&found)
+		err := app.DB.QueryRow("SELECT 1 FROM all_view WHERE shop_id = ? AND name = ? AND long_url = ? AND original_price = ? AND discounted_price = ? AND spirit_type = ?", o.Shop, o.Name, o.Url, o.Original_price, o.Discounted_price, o.Spirit_type).Scan(&found)
 
 		/*
 		 */
@@ -105,6 +109,7 @@ func (app *App) save_offer(W []Angebot) error {
 				_, err = stmt.Exec(o.Shop, o.Name, o.Url, o.Abv, o.Volume, o.Original_price, o.Discounted_price, o.Base_price, o.Valid_until, o.Image_url, o.Website, o.Spirit_type, app.Now)
 			}
 			if err != nil {
+				Debug(err, "Save Offer: Inserting offer failed")
 				return err
 			}
 
@@ -115,12 +120,14 @@ func (app *App) save_offer(W []Angebot) error {
 	return nil
 }
 
-func (app *App) remove_expired(W []Angebot, shop_id int) error {
+func (app *App) remove_expired(W []Angebot, shop Shop) error {
 
-	query := `SELECT id, shop, name, url, original_price, discounted_price FROM all_view WHERE shop = ? AND created_at < ?`
+	query := `SELECT id, shop, name, url, original_price, discounted_price FROM angebot WHERE shop = ? AND created_at < ?
+		AND (valid_until IS NULL OR valid_until > ?)`
 
-	rows, err := app.DB.Queryx(query, shop_id, app.Now)
+	rows, err := app.DB.Queryx(query, shop.Id, app.Now, app.Now)
 	if err != nil {
+		Debug(err, "Remove expired: Query failed")
 		return err
 	}
 	defer rows.Close()
@@ -130,16 +137,20 @@ func (app *App) remove_expired(W []Angebot, shop_id int) error {
 		err = rows.StructScan(&offer_db)
 
 		if err != nil {
+			Debug(err, "Remove expired: Struct scan failed")
 			return err
 		}
 
 		if !app.offer_contains(W, offer_db) {
+			DebugOffer(offer_db, "Contains not - Set to expire")
 			expire_query := `UPDATE angebot SET valid_until = ? WHERE id = ?`
 			_, err = app.DB.Exec(expire_query, app.Now, offer_db.Id)
 			if err != nil {
+				Debug(err, "Remove expired: Update failed")
 				return err
 			}
 		}
+		DebugOffer(offer_db, "Contains! DOES NOT EXPIRE!")
 	}
 
 	return nil
@@ -154,8 +165,6 @@ func (app *App) offer_contains(W []Angebot, offer_db Angebot) bool {
 		if (v.Shop == offer_db.Shop) && (v.Name == offer_db.Name) && (v.Original_price == offer_db.Original_price) && (v.Discounted_price == offer_db.Discounted_price) {
 
 			if app.Config.Debug {
-				log.Println("Contains: " + v.Name)
-				log.Println("")
 			}
 
 			return true
@@ -164,8 +173,7 @@ func (app *App) offer_contains(W []Angebot, offer_db Angebot) bool {
 	}
 
 	if app.Config.Debug {
-		log.Println("Contains not: " + offer_db.Name)
-		log.Println("")
+		log.Debug("Contains not: " + offer_db.Name)
 	}
 
 	return false
diff --git a/crawler/log.go b/crawler/log.go
new file mode 100644
index 0000000..4b9f374
--- /dev/null
+++ b/crawler/log.go
@@ -0,0 +1,79 @@
+package main
+
+import (
+	log "github.com/Sirupsen/logrus"
+)
+
+func init() {
+	log.SetLevel(log.DebugLevel)
+}
+
+func Fatal(err error, msg string) {
+	log.WithFields(
+		log.Fields{
+			"error": err.Error(),
+		},
+	).Fatal(msg)
+}
+
+func Println(err error, msg string) {
+	if err != nil {
+		log.WithFields(
+			log.Fields{
+				"error": err.Error(),
+			},
+		).Println(msg)
+	} else {
+		log.Println(msg)
+	}
+}
+
+func PrintlnOffer(offer Angebot, msg string) {
+
+	log.WithFields(
+		log.Fields{
+			"Name":             offer.Name,
+			"Shop":             offer.Shop,
+			"ABV":              offer.Abv,
+			"Volume":           offer.Volume,
+			"Url":              offer.Url,
+			"Original Price":   offer.Original_price,
+			"Discounted Price": offer.Discounted_price,
+			"Base Price":       offer.Base_price,
+			"Image_url":        offer.Image_url,
+			"Spirit Type":      offer.Spirit_type,
+			"Valid Until":      offer.Valid_until,
+		},
+	).Println(msg)
+}
+
+func Debug(err error, msg string) {
+	if err != nil {
+		log.WithFields(
+			log.Fields{
+				"error": err.Error(),
+			},
+		).Debug(msg)
+	} else {
+		log.Debug(msg)
+	}
+}
+
+func DebugOffer(offer Angebot, msg string) {
+
+	log.WithFields(
+		log.Fields{
+			"Name":             offer.Name,
+			"Shop":             offer.Shop,
+			"ABV":              offer.Abv,
+			"Volume":           offer.Volume,
+			"Url":              offer.Url,
+			"Original Price":   offer.Original_price,
+			"Discounted Price": offer.Discounted_price,
+			"Base Price":       offer.Base_price,
+			"Image_url":        offer.Image_url,
+			"Spirit Type":      offer.Spirit_type,
+			"Valid Until":      offer.Valid_until,
+		},
+	).Debug(msg)
+}
diff --git a/crawler/main.go b/crawler/main.go
index f4a90b7..11dfd4d 100644
--- a/crawler/main.go
+++ b/crawler/main.go
@@ -3,7 +3,7 @@ package main
 import (
 	"encoding/json"
 	"fmt"
-	"log"
+	log "github.com/Sirupsen/logrus"
 	"time"
 
 	_ "database/sql"
@@ -64,32 +64,32 @@ func main() {
 	} else {
 
 		if app.Config.Debug {
-			log.Println(app.Config.DBUser + ":" + app.Config.DBPassword + "@tcp(" + app.Config.DBHost + ":" + app.Config.DBPort + ")/" + app.Config.DBDBName + app.Config.DBOptions)
+			log.Debug(app.Config.DBUser + ":" + app.Config.DBPassword + "@tcp(" + app.Config.DBHost + ":" + app.Config.DBPort + ")/" + app.Config.DBDBName + app.Config.DBOptions)
 		}
 		app.DB, err = sqlx.Connect(app.Config.DBDriver, app.Config.DBUser+":"+app.Config.DBPassword+"@tcp("+app.Config.DBHost+":"+app.Config.DBPort+")/"+app.Config.DBDBName+app.Config.DBOptions)
 	}
 	if err != nil {
-		log.Fatal(err)
+		Fatal(err, "Cannot connect to database")
 	}
 
 	if err = app.DB.Ping(); err != nil {
-		log.Fatal(err)
+		Fatal(err, "No connection to database")
 	}
 	defer app.DB.Close()
 
 	err = app.createTables()
 	if err != nil {
-		log.Fatal(err)
+		Fatal(err, "Creating table failed")
 	}
 
 	err = app.insertShops()
 	if err != nil {
-		log.Fatal(err)
+		Fatal(err, "Inserting shops failed")
 	}
 
 	shops, err := app.getShops()
 	if err != nil {
-		log.Fatal(err)
+		Fatal(err, "Getting shops failed")
 	}
 
 	app.ScrapeHTML(shops)
@@ -97,7 +97,7 @@ func main() {
 	// short url
 	err = app.post_process()
 	if err != nil {
-		log.Fatal(err)
+		Fatal(err, "Post processing failed")
 	}
 }
 
diff --git a/crawler/post_process.go b/crawler/post_process.go
index e527911..dfd7861 100644
--- a/crawler/post_process.go
+++ b/crawler/post_process.go
@@ -2,9 +2,10 @@ package main
 
 import (
 	"io/ioutil"
-	"log"
 	"net/http"
 	"net/url"
+
+	log "github.com/Sirupsen/logrus"
 )
 
 func (app *App) post_process() error {
@@ -37,7 +38,7 @@ func (app *App) short_url() error {
 		polr_url := app.Config.Polr_URL + "?" + v.Encode()
 
 		if app.Config.Debug {
-			log.Println("DEBUG: polr_url: " + polr_url + " ( " + offer_db.Url + " )")
+			log.Debug("polr_url: " + polr_url + " ( " + offer_db.Url + " )")
 		}
 		resp, err := http.Get(polr_url)
 		if err != nil {
@@ -52,7 +53,7 @@ func (app *App) short_url() error {
 
 		offer_db.Short_url = string(short_url)
 		if app.Config.Debug {
-			log.Println("DEBUG: short_url: " + string(short_url) + " ( " + offer_db.Url + " )")
+			log.Debug("short_url: " + string(short_url) + " ( " + offer_db.Url + " )")
 		}
 
 		Angebote = append(Angebote, offer_db)
diff --git a/crawler/sanitize.go b/crawler/sanitize.go
index 7a2ff58..949e0f0 100644
--- a/crawler/sanitize.go
+++ b/crawler/sanitize.go
@@ -1,7 +1,7 @@
 package main
 
 import (
-	"log"
+	log "github.com/Sirupsen/logrus"
 	"regexp"
 	"strings"
 )
diff --git a/crawler/scrape.go b/crawler/scrape.go
index 18736fe..ced1d98 100644
--- a/crawler/scrape.go
+++ b/crawler/scrape.go
@@ -1,7 +1,7 @@
 package main
 
 import (
-	"log"
+	log "github.com/Sirupsen/logrus"
 )
 
 func (app *App) ScrapeHTML(shops []Shop) {
@@ -34,11 +34,11 @@ func (app *App) Scrape(shop Shop, wait chan bool) {
 
 	err = app.save_offer(W)
 	if err != nil {
-		log.Fatal(err)
+		Fatal(err, "Saving offers failed")
 	}
-	err = app.remove_expired(W, shop.Id)
+	err = app.remove_expired(W, shop)
 	if err != nil {
-		log.Fatal(err)
+		Fatal(err, "Removing expired offers failed")
 	}
 
 	wait <- true
diff --git a/crawler/shop_bottleworld.go b/crawler/shop_bottleworld.go
index 06082d4..2eea3e5 100644
--- a/crawler/shop_bottleworld.go
+++ b/crawler/shop_bottleworld.go
@@ -1,7 +1,7 @@
 package main
 
 import (
-	"log"
+	log "github.com/Sirupsen/logrus"
 	"regexp"
 	"strings"
 
diff --git a/crawler/shop_mcwhisky.go b/crawler/shop_mcwhisky.go
index 2d4c0db..165d944 100644
--- a/crawler/shop_mcwhisky.go
+++ b/crawler/shop_mcwhisky.go
@@ -1,7 +1,7 @@
 package main
 
 import (
-	"log"
+	log "github.com/Sirupsen/logrus"
 	"regexp"
 	//	"strings"
 
diff --git a/crawler/shop_rumundco.go b/crawler/shop_rumundco.go
index cd67c0f..e1516ba 100644
--- a/crawler/shop_rumundco.go
+++ b/crawler/shop_rumundco.go
@@ -1,7 +1,6 @@
 package main
 
 import (
-	"log"
 	"regexp"
 	"strings"
 
@@ -33,7 +32,7 @@ func ScrapeRumundCo(shop Shop) []Angebot {
 
 		matched, err := regexp.MatchString("verfügbar", e.ChildText(".delivery-status"))
 		if err != nil {
-			log.Fatal(err)
+			Fatal(err, "Rum & Co: Verfügbar regex failed")
 		}
 		if !matched {
 			return
@@ -42,41 +41,72 @@ func ScrapeRumundCo(shop Shop) []Angebot {
 		W.Name = whisky_name
 		W.Url = whisky_url
 
+		r_abv, err := regexp.Compile("[0-9]+([,.][0-9])?( )*(%|([vV]ol))")
+		if err != nil {
+			Fatal(err, "Rum & Co: Abv regex failed")
+		}
+		abv_noisy := r_abv.FindString(whisky_name)
+
 		e.ForEach(".price_wrapper", func(i int, e *colly.HTMLElement) {
 			regular_price := e.ChildText("del.value")
 			if "" == regular_price {
+				PrintlnOffer(W, "Rum & Co: No regular price found")
 				return
 			}
 			W.Original_price, err = convert_price(regular_price)
 			if err != nil {
-				log.Fatal(err)
+				Fatal(err, "Rum & Co: Original price: Convert price failed")
 			}
 			W.Discounted_price, err = convert_price(e.ChildText(".price-value"))
 			if err != nil {
-				log.Fatal(err)
+				Fatal(err, "Rum & Co: Discounted price: Convert price failed")
 			}
 
 			e.ForEach(".base_price", func(i int, e *colly.HTMLElement) {
 				price_per_litre_noisy := e.ChildText(".value")
 				W.Base_price, err = sanitize_base_price(price_per_litre_noisy)
 				if err != nil {
-					log.Fatal(err)
+					Fatal(err, "Rum & Co: Base price: Sanitizing base price failed")
 				}
 			})
 
 		})
 
-		W.Image_url = "https://www.rumundco.de/" + e.ChildAttr("img", "src")
+		// Rum & Co uses pagespeed
+		image_url_noisy := e.ChildAttr("img", "src")
+
+		if strings.Contains(image_url_noisy, "pagespeed") {
+			r_pagespeed, err := regexp.Compile(`jpg(\.pagespeed.+)$`)
+			if err != nil {
+				Fatal(err, "Rum & Co: Pagespeed regexp failed")
+			}
+			image_url_noisy_slice := r_pagespeed.FindStringSubmatch(image_url_noisy)
+			if len(image_url_noisy_slice) < 2 {
+				PrintlnOffer(W, "Rum & Co: (Pagespeed) Image URL not found")
+				return
+			}
+			image_url_noisy = strings.Replace(image_url_noisy, image_url_noisy_slice[1], "", 1)
+		}
+
+		W.Image_url = "https://www.rumundco.de/" + image_url_noisy
 
 		e.Request.Visit(W.Url)
 
 		W.Volume = get_volume(e)
 
 		if W.Volume == 0 {
+			PrintlnOffer(W, "Rum & Co: No Volume found")
 			return
 		}
 
-		W.Abv = get_abv(e)
+		if "" == abv_noisy {
+			W.Abv = get_abv(e)
+		} else {
+			W.Abv, err = extract_abv(abv_noisy)
+			if err != nil {
+				Fatal(err, "Rum & Co: Base price: Extracting ABV failed")
+			}
+		}
 
 		W.Shop = shop.Id
 		W.Spirit_type = "Whisky"
diff --git a/crawler/shop_whic.go b/crawler/shop_whic.go
index b4f9562..e2bb6b9 100644
--- a/crawler/shop_whic.go
+++ b/crawler/shop_whic.go
@@ -1,7 +1,7 @@
 package main
 
 import (
-	"log"
+	log "github.com/Sirupsen/logrus"
 	"regexp"
 	"strings"
 
diff --git a/crawler/shop_whiskyde.go b/crawler/shop_whiskyde.go
index 6038c11..e635546 100644
--- a/crawler/shop_whiskyde.go
+++ b/crawler/shop_whiskyde.go
@@ -1,7 +1,7 @@
 package main
 
 import (
-	"log"
+	log "github.com/Sirupsen/logrus"
 	"strings"
 
 	"github.com/gocolly/colly"
diff --git a/crawler/shop_whiskysitenl.go b/crawler/shop_whiskysitenl.go
index 656cf18..a555123 100644
--- a/crawler/shop_whiskysitenl.go
+++ b/crawler/shop_whiskysitenl.go
@@ -1,7 +1,7 @@
 package main
 
 import (
-	"log"
+	log "github.com/Sirupsen/logrus"
 	"regexp"
 	"strings"
 
diff --git a/crawler/shop_whiskyworld.go b/crawler/shop_whiskyworld.go
index c0fb7b6..9b968a3 100644
--- a/crawler/shop_whiskyworld.go
+++ b/crawler/shop_whiskyworld.go
@@ -1,7 +1,7 @@
 package main
 
 import (
-	"log"
+	log "github.com/Sirupsen/logrus"
 	"strings"
 
 	"github.com/gocolly/colly"
diff --git a/crawler/shop_whiskyzone.go b/crawler/shop_whiskyzone.go
index 5809b7e..c07c14f 100644
--- a/crawler/shop_whiskyzone.go
+++ b/crawler/shop_whiskyzone.go
@@ -1,7 +1,6 @@
 package main
 
 import (
-	"log"
 	"strings"
 
 	"github.com/gocolly/colly"
@@ -38,17 +37,20 @@ func ScrapeWhiskyzone(shop Shop) []Angebot {
 
 		e.Request.Visit(W.Url)
 
+		if "sold_out" == e.Request.Ctx.Get("sold_out") {
+			PrintlnOffer(W, "Whiskyzone: Sold out")
+			return
+		}
+
 		var err error
 		W.Discounted_price, err = convert_price(e.Request.Ctx.Get("discounted_price"))
 		if err != nil {
-			log.Println("Discounted_price failed")
-			log.Fatal(err)
+			Fatal(err, "Whiskyzone: Convert discounted price failed")
 		}
 
 		W.Original_price, err = convert_price(e.Request.Ctx.Get("original_price"))
 		if err != nil {
-			log.Println("Original_price failed")
-			log.Fatal(err)
+			Fatal(err, "Whiskyzone: Convert original price failed")
 		}
 
 		W.Volume = get_volume(e)
@@ -110,6 +112,16 @@ func ScrapeWhiskyzone(shop Shop) []Angebot {
 		e.Request.Ctx.Put("website", string(e.Response.Body))
 	})
 
+	// Product not found
+	c.OnHTML(".detail-error--headline", func(e *colly.HTMLElement) {
+		if e.Request.URL.String() == Shop_url {
+			return
+		}
+
+		e.Request.Ctx.Put("sold_out", "sold_out")
+
+	})
+
 	c.Visit(Shop_url)
 
 	return Whiskys
diff --git a/crawler/shops.go b/crawler/shops.go
index 9cad602..8cfd9f4 100644
--- a/crawler/shops.go
+++ b/crawler/shops.go
@@ -1,7 +1,7 @@
 package main
 
 import (
-	"log"
+	log "github.com/Sirupsen/logrus"
 )
 
 func (app *App) insertShops() error {
@@ -112,7 +112,7 @@ func (app *App) getShops() ([]Shop, error) {
 			return []Shop{}, err
 		}
 		if app.Config.Debug {
-			log.Println("Appending: " + shop.Name)
+			log.Println("Crawling: " + shop.Name)
 		}
 
 		Shops = append(Shops, shop)
diff --git a/crawler/utility.go b/crawler/utility.go
index c3daeb1..98587c9 100644
--- a/crawler/utility.go
+++ b/crawler/utility.go
@@ -1,7 +1,7 @@
 package main
 
 import (
-	"log"
+	log "github.com/Sirupsen/logrus"
 	"regexp"
 	"strconv"
 	"strings"
@@ -21,56 +21,56 @@ func stringInSlice(a string, list []string) bool {
 func detect_spirit_type(name string) string {
 	matched, err := regexp.MatchString(`(^|\s)Gin(\s|$)`, name)
 	if err != nil {
-		log.Fatal(err)
+		Fatal(err, "Gin regex failed")
 	}
 	if matched {
 		return "Gin"
 	}
 	matched, err = regexp.MatchString(`(^|\s)Rh?um(\s|$)`, name)
 	if err != nil {
-		log.Fatal(err)
+		Fatal(err, "Rum regex failed")
 	}
 	if matched {
 		return "Rum"
 	}
 	matched, err = regexp.MatchString(`(^|\s)[VW]odka(\s|$)`, name)
 	if err != nil {
-		log.Fatal(err)
+		Fatal(err, "Wodka regex failed")
 	}
 	if matched {
 		return "Wodka"
 	}
 	matched, err = regexp.MatchString(`(^|\s)Whiske?y(\s|$)`, name)
 	if err != nil {
-		log.Fatal(err)
+		Fatal(err, "Whisky regex failed")
 	}
 	if matched {
 		return "Whisky"
 	}
 	matched, err = regexp.MatchString(`(^|\s)Champagner(\s|$)`, name)
 	if err != nil {
-		log.Fatal(err)
+		Fatal(err, "Champagner regex failed")
 	}
 	if matched {
 		return "Champagner"
 	}
 	matched, err = regexp.MatchString(`(^|\s)Cognac(\s|$)`, name)
 	if err != nil {
-		log.Fatal(err)
+		Fatal(err, "Cognac regex failed")
 	}
 	if matched {
 		return "Cognac"
 	}
 	matched, err = regexp.MatchString(`(^|\s)Grappa(\s|$)`, name)
 	if err != nil {
-		log.Fatal(err)
+		Fatal(err, "Grappa regex failed")
 	}
 	if matched {
 		return "Grappa"
 	}
 	matched, err = regexp.MatchString(`(^|\s)Likör(\s|$)`, name)
 	if err != nil {
-		log.Fatal(err)
+		Fatal(err, "Likör regex failed")
 	}
 	if matched {
 		return "Likör"
@@ -82,18 +82,19 @@ func detect_spirit_type(name string) string {
 func extract_volume(volume string) (float32, error) {
 	r_liter, err := regexp.Compile(`[0-9]+([,.][0-9]+)?( )?[lL](iter)?`)
 	if err != nil {
-		log.Fatal(err)
+		Fatal(err, "Extract volume regex failed")
 	}
 	volume_noisy := r_liter.FindString(volume)
 
 	r_liter2, err := regexp.Compile(`[0-9]+([,.][0-9]+)?`)
 	if err != nil {
-		log.Fatal(err)
+		Fatal(err, "2nd extract volume regex failed")
 	}
 	volume_noisy = r_liter2.FindString(volume_noisy)
 	volume_noisy = strings.Replace(volume_noisy, ",", ".", 1)
 	volume64, err := strconv.ParseFloat(volume_noisy, 32)
 	if err != nil {
+		Println(err, "Parsing volume to float failed")
 		return 0, err
 	}
 
@@ -114,12 +115,13 @@ func extract_abv(abv_noisy string) (float32, error) {
 	abv_noisy = strings.TrimSpace(abv_noisy)
 	r_abv, err := regexp.Compile(`[0-9]+([,.][0-9]+)?`)
 	if err != nil {
-		log.Fatal(err)
+		Fatal(err, "Extract abv regex failed")
 	}
 	abv_noisy = r_abv.FindString(abv_noisy)
 
 	abv64, err := strconv.ParseFloat(abv_noisy, 32)
 	if err != nil {
+		Println(err, "Parsing abv to float failed")
 		return 0, err
 	}
 
@@ -135,6 +137,7 @@ func get_volume(e *colly.HTMLElement) float32 {
 
 	matched, err := regexp.MatchString(`[lL](iter)?`, volume_noisy)
 	if err != nil {
+		Fatal(err, "Get volume regex failed")
 		log.Fatal(err)
 	}
 	if !matched {
@@ -145,7 +148,7 @@ func get_volume(e *colly.HTMLElement) float32 {
 	volume, err := extract_volume(volume_noisy)
 	if err != nil {
 		log.Println("get_volume: " + volume_noisy)
-		log.Fatal(err)
+		Fatal(err, "Get Volume: Extract Volume failed")
 	}
 
 	return volume
@@ -166,7 +169,7 @@ func get_abv(e *colly.HTMLElement) float32 {
 	abv, err := extract_abv(abv_noisy)
 	if err != nil {
 		log.Println("get_abv: " + abv_noisy)
-		log.Fatal(err)
+		Fatal(err, "Get ABV: Extract ABV failed")
 	}
 
 	return abv
@@ -186,7 +189,7 @@ func get_base_price(e *colly.HTMLElement) int {
 	base_price, err := sanitize_base_price(base_price_noisy)
 	if err != nil {
 		log.Println("get_base_price: " + base_price_noisy)
-		log.Fatal(err)
+		Fatal(err, "Get base price: sanitize base price failed")
 	}
 
 	return base_price