summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--crawler/convert_price.go3
-rw-r--r--crawler/database.go30
-rw-r--r--crawler/log.go79
-rw-r--r--crawler/main.go16
-rw-r--r--crawler/post_process.go7
-rw-r--r--crawler/sanitize.go2
-rw-r--r--crawler/scrape.go8
-rw-r--r--crawler/shop_bottleworld.go2
-rw-r--r--crawler/shop_mcwhisky.go2
-rw-r--r--crawler/shop_rumundco.go44
-rw-r--r--crawler/shop_whic.go2
-rw-r--r--crawler/shop_whiskyde.go2
-rw-r--r--crawler/shop_whiskysitenl.go2
-rw-r--r--crawler/shop_whiskyworld.go2
-rw-r--r--crawler/shop_whiskyzone.go22
-rw-r--r--crawler/shops.go4
-rw-r--r--crawler/utility.go33
17 files changed, 196 insertions, 64 deletions
diff --git a/crawler/convert_price.go b/crawler/convert_price.go
index f1c88b4..b28639f 100644
--- a/crawler/convert_price.go
+++ b/crawler/convert_price.go
@@ -2,7 +2,6 @@ package main
import (
"errors"
- "log"
"regexp"
"strconv"
"strings"
@@ -98,7 +97,7 @@ func convert_price(price string) (int, error) {
*/
price_int, err := strconv.Atoi(price)
if err != nil {
- log.Println(price)
+ Println(err, price)
return 0, err
}
diff --git a/crawler/database.go b/crawler/database.go
index 1f38ca1..2608c59 100644
--- a/crawler/database.go
+++ b/crawler/database.go
@@ -2,8 +2,9 @@ package main
import (
"database/sql"
- "log"
"strings"
+
+ log "github.com/Sirupsen/logrus"
)
func (app *App) createTables() error {
@@ -59,8 +60,10 @@ func (app *App) createTables() error {
view_query := `CREATE OR REPLACE VIEW ` + v + `_view AS
SELECT
- angebot.id, angebot.name, angebot.abv, angebot.volume, angebot.url as long_url, angebot.short_url as url,original_price/100 as original_price, discounted_price/100 as discounted_price, angebot.base_price/100 as base_price, image_url,
- shop.name as shop, shop.short_url as shop_url, shop.shipping_costs/100 as shipping_costs, shop.free_shipping, ROUND(100-((discounted_price/original_price)*100)) AS procent, spirit_type, created_at
+ angebot.id, angebot.name, angebot.abv, angebot.volume, angebot.url as long_url, angebot.short_url as url, spirit_type,
+ original_price/100 as original_price, discounted_price/100 as discounted_price, angebot.base_price/100 as base_price, image_url,
+ shop.name as shop, shop.id as shop_id, shop.short_url as shop_url, shop.shipping_costs/100 as shipping_costs, shop.free_shipping,
+ ROUND(100-((discounted_price/original_price)*100)) AS procent, created_at
FROM angebot
JOIN shop ON angebot.shop = shop.id
WHERE
@@ -80,6 +83,7 @@ func (app *App) save_offer(W []Angebot) error {
stmt, err := app.DB.Prepare(query)
if err != nil {
+ Debug(err, "Save Offer: Preparing query failed")
return err
}
defer stmt.Close()
@@ -92,7 +96,7 @@ func (app *App) save_offer(W []Angebot) error {
continue
}
- err := app.DB.QueryRow("SELECT 1 FROM angebot WHERE shop = ? AND name = ? AND url = ? AND original_price = ? AND discounted_price = ? AND spirit_type = ?", o.Shop, o.Name, o.Url, o.Original_price, o.Discounted_price, o.Spirit_type).Scan(&found)
+ err := app.DB.QueryRow("SELECT 1 FROM all_view WHERE shop_id = ? AND name = ? AND long_url = ? AND original_price = ? AND discounted_price = ? AND spirit_type = ?", o.Shop, o.Name, o.Url, o.Original_price, o.Discounted_price, o.Spirit_type).Scan(&found)
/*
*/
@@ -105,6 +109,7 @@ func (app *App) save_offer(W []Angebot) error {
_, err = stmt.Exec(o.Shop, o.Name, o.Url, o.Abv, o.Volume, o.Original_price, o.Discounted_price, o.Base_price, o.Valid_until, o.Image_url, o.Website, o.Spirit_type, app.Now)
}
if err != nil {
+ Debug(err, "Save Offer: Inserting offer failed")
return err
}
@@ -115,12 +120,14 @@ func (app *App) save_offer(W []Angebot) error {
return nil
}
-func (app *App) remove_expired(W []Angebot, shop_id int) error {
+func (app *App) remove_expired(W []Angebot, shop Shop) error {
- query := `SELECT id, shop, name, url, original_price, discounted_price FROM all_view WHERE shop = ? AND created_at < ?`
+ query := `SELECT id, shop, name, url, original_price, discounted_price FROM angebot WHERE shop = ? AND created_at < ?
+ AND (valid_until IS NULL OR valid_until > ?)`
- rows, err := app.DB.Queryx(query, shop_id, app.Now)
+ rows, err := app.DB.Queryx(query, shop.Id, app.Now, app.Now)
if err != nil {
+ Debug(err, "Remove expired: Query failed")
return err
}
defer rows.Close()
@@ -130,16 +137,20 @@ func (app *App) remove_expired(W []Angebot, shop_id int) error {
err = rows.StructScan(&offer_db)
if err != nil {
+ Debug(err, "Remove expired: Struct scan failed")
return err
}
if !app.offer_contains(W, offer_db) {
+ DebugOffer(offer_db, "Contains not - Set to expire")
expire_query := `UPDATE angebot SET valid_until = ? WHERE id = ?`
_, err = app.DB.Exec(expire_query, app.Now, offer_db.Id)
if err != nil {
+ Debug(err, "Remove expired: Update failed")
return err
}
}
+ DebugOffer(offer_db, "Contains! DOES NOT EXPIRE!")
}
return nil
@@ -154,8 +165,6 @@ func (app *App) offer_contains(W []Angebot, offer_db Angebot) bool {
if (v.Shop == offer_db.Shop) && (v.Name == offer_db.Name) && (v.Original_price == offer_db.Original_price) && (v.Discounted_price == offer_db.Discounted_price) {
if app.Config.Debug {
- log.Println("Contains: " + v.Name)
- log.Println("")
}
return true
@@ -164,8 +173,7 @@ func (app *App) offer_contains(W []Angebot, offer_db Angebot) bool {
}
if app.Config.Debug {
- log.Println("Contains not: " + offer_db.Name)
- log.Println("")
+ log.Debug("Contains not: " + offer_db.Name)
}
return false
diff --git a/crawler/log.go b/crawler/log.go
new file mode 100644
index 0000000..4b9f374
--- /dev/null
+++ b/crawler/log.go
@@ -0,0 +1,79 @@
+package main
+
+import (
+ log "github.com/Sirupsen/logrus"
+)
+
+func init() {
+ log.SetLevel(log.DebugLevel)
+}
+
+func Fatal(err error, msg string) {
+ log.WithFields(
+ log.Fields{
+ "error": err.Error(),
+ },
+ ).Fatal(msg)
+}
+
+func Println(err error, msg string) {
+ if err != nil {
+ log.WithFields(
+ log.Fields{
+ "error": err.Error(),
+ },
+ ).Println(msg)
+ } else {
+ log.Println(msg)
+ }
+}
+
+func PrintlnOffer(offer Angebot, msg string) {
+
+ log.WithFields(
+ log.Fields{
+ "Name": offer.Name,
+ "Shop": offer.Shop,
+ "ABV": offer.Abv,
+ "Volume": offer.Volume,
+ "Url": offer.Url,
+ "Original Price": offer.Original_price,
+ "Discounted Price": offer.Discounted_price,
+ "Base Price": offer.Base_price,
+ "Image_url": offer.Image_url,
+ "Spirit Type": offer.Spirit_type,
+ "Valid Until": offer.Valid_until,
+ },
+ ).Println(msg)
+}
+
+func Debug(err error, msg string) {
+ if err != nil {
+ log.WithFields(
+ log.Fields{
+ "error": err.Error(),
+ },
+ ).Debug(msg)
+ } else {
+ log.Debug(msg)
+ }
+}
+
+func DebugOffer(offer Angebot, msg string) {
+
+ log.WithFields(
+ log.Fields{
+ "Name": offer.Name,
+ "Shop": offer.Shop,
+ "ABV": offer.Abv,
+ "Volume": offer.Volume,
+ "Url": offer.Url,
+ "Original Price": offer.Original_price,
+ "Discounted Price": offer.Discounted_price,
+ "Base Price": offer.Base_price,
+ "Image_url": offer.Image_url,
+ "Spirit Type": offer.Spirit_type,
+ "Valid Until": offer.Valid_until,
+ },
+ ).Debug(msg)
+}
diff --git a/crawler/main.go b/crawler/main.go
index f4a90b7..11dfd4d 100644
--- a/crawler/main.go
+++ b/crawler/main.go
@@ -3,7 +3,7 @@ package main
import (
"encoding/json"
"fmt"
- "log"
+ log "github.com/Sirupsen/logrus"
"time"
_ "database/sql"
@@ -64,32 +64,32 @@ func main() {
} else {
if app.Config.Debug {
- log.Println(app.Config.DBUser + ":" + app.Config.DBPassword + "@tcp(" + app.Config.DBHost + ":" + app.Config.DBPort + ")/" + app.Config.DBDBName + app.Config.DBOptions)
+ log.Debug(app.Config.DBUser + ":" + app.Config.DBPassword + "@tcp(" + app.Config.DBHost + ":" + app.Config.DBPort + ")/" + app.Config.DBDBName + app.Config.DBOptions)
}
app.DB, err = sqlx.Connect(app.Config.DBDriver, app.Config.DBUser+":"+app.Config.DBPassword+"@tcp("+app.Config.DBHost+":"+app.Config.DBPort+")/"+app.Config.DBDBName+app.Config.DBOptions)
}
if err != nil {
- log.Fatal(err)
+ Fatal(err, "Cannot connect to database")
}
if err = app.DB.Ping(); err != nil {
- log.Fatal(err)
+ Fatal(err, "No connection to database")
}
defer app.DB.Close()
err = app.createTables()
if err != nil {
- log.Fatal(err)
+ Fatal(err, "Creating table failed")
}
err = app.insertShops()
if err != nil {
- log.Fatal(err)
+ Fatal(err, "Inserting shops failed")
}
shops, err := app.getShops()
if err != nil {
- log.Fatal(err)
+ Fatal(err, "Getting shops failed")
}
app.ScrapeHTML(shops)
@@ -97,7 +97,7 @@ func main() {
// short url
err = app.post_process()
if err != nil {
- log.Fatal(err)
+ Fatal(err, "Post processing failed")
}
}
diff --git a/crawler/post_process.go b/crawler/post_process.go
index e527911..dfd7861 100644
--- a/crawler/post_process.go
+++ b/crawler/post_process.go
@@ -2,9 +2,10 @@ package main
import (
"io/ioutil"
- "log"
"net/http"
"net/url"
+
+ log "github.com/Sirupsen/logrus"
)
func (app *App) post_process() error {
@@ -37,7 +38,7 @@ func (app *App) short_url() error {
polr_url := app.Config.Polr_URL + "?" + v.Encode()
if app.Config.Debug {
- log.Println("DEBUG: polr_url: " + polr_url + " ( " + offer_db.Url + " )")
+ log.Debug("polr_url: " + polr_url + " ( " + offer_db.Url + " )")
}
resp, err := http.Get(polr_url)
if err != nil {
@@ -52,7 +53,7 @@ func (app *App) short_url() error {
offer_db.Short_url = string(short_url)
if app.Config.Debug {
- log.Println("DEBUG: short_url: " + string(short_url) + " ( " + offer_db.Url + " )")
+ log.Debug("short_url: " + string(short_url) + " ( " + offer_db.Url + " )")
}
Angebote = append(Angebote, offer_db)
diff --git a/crawler/sanitize.go b/crawler/sanitize.go
index 7a2ff58..949e0f0 100644
--- a/crawler/sanitize.go
+++ b/crawler/sanitize.go
@@ -1,7 +1,7 @@
package main
import (
- "log"
+ log "github.com/Sirupsen/logrus"
"regexp"
"strings"
)
diff --git a/crawler/scrape.go b/crawler/scrape.go
index 18736fe..ced1d98 100644
--- a/crawler/scrape.go
+++ b/crawler/scrape.go
@@ -1,7 +1,7 @@
package main
import (
- "log"
+ log "github.com/Sirupsen/logrus"
)
func (app *App) ScrapeHTML(shops []Shop) {
@@ -34,11 +34,11 @@ func (app *App) Scrape(shop Shop, wait chan bool) {
err = app.save_offer(W)
if err != nil {
- log.Fatal(err)
+ Fatal(err, "Saving offers failed")
}
- err = app.remove_expired(W, shop.Id)
+ err = app.remove_expired(W, shop)
if err != nil {
- log.Fatal(err)
+ Fatal(err, "Removing expired offers failed")
}
wait <- true
diff --git a/crawler/shop_bottleworld.go b/crawler/shop_bottleworld.go
index 06082d4..2eea3e5 100644
--- a/crawler/shop_bottleworld.go
+++ b/crawler/shop_bottleworld.go
@@ -1,7 +1,7 @@
package main
import (
- "log"
+ log "github.com/Sirupsen/logrus"
"regexp"
"strings"
diff --git a/crawler/shop_mcwhisky.go b/crawler/shop_mcwhisky.go
index 2d4c0db..165d944 100644
--- a/crawler/shop_mcwhisky.go
+++ b/crawler/shop_mcwhisky.go
@@ -1,7 +1,7 @@
package main
import (
- "log"
+ log "github.com/Sirupsen/logrus"
"regexp"
// "strings"
diff --git a/crawler/shop_rumundco.go b/crawler/shop_rumundco.go
index cd67c0f..e1516ba 100644
--- a/crawler/shop_rumundco.go
+++ b/crawler/shop_rumundco.go
@@ -1,7 +1,6 @@
package main
import (
- "log"
"regexp"
"strings"
@@ -33,7 +32,7 @@ func ScrapeRumundCo(shop Shop) []Angebot {
matched, err := regexp.MatchString("verfügbar", e.ChildText(".delivery-status"))
if err != nil {
- log.Fatal(err)
+ Fatal(err, "Rum & Co: Verfügbar regex failed")
}
if !matched {
return
@@ -42,41 +41,72 @@ func ScrapeRumundCo(shop Shop) []Angebot {
W.Name = whisky_name
W.Url = whisky_url
+ r_abv, err := regexp.Compile("[0-9]+([,.][0-9])?( )*(%|([vV]ol))")
+ if err != nil {
+ Fatal(err, "Rum & Co: Abv regex failed")
+ }
+ abv_noisy := r_abv.FindString(whisky_name)
+
e.ForEach(".price_wrapper", func(i int, e *colly.HTMLElement) {
regular_price := e.ChildText("del.value")
if "" == regular_price {
+ PrintlnOffer(W, "Rum & Co: No regular price found")
return
}
W.Original_price, err = convert_price(regular_price)
if err != nil {
- log.Fatal(err)
+ Fatal(err, "Rum & Co: Original price: Convert price failed")
}
W.Discounted_price, err = convert_price(e.ChildText(".price-value"))
if err != nil {
- log.Fatal(err)
+ Fatal(err, "Rum & Co: Discounted price: Convert price failed")
}
e.ForEach(".base_price", func(i int, e *colly.HTMLElement) {
price_per_litre_noisy := e.ChildText(".value")
W.Base_price, err = sanitize_base_price(price_per_litre_noisy)
if err != nil {
- log.Fatal(err)
+ Fatal(err, "Rum & Co: Base price: Sanitizing base price failed")
}
})
})
- W.Image_url = "https://www.rumundco.de/" + e.ChildAttr("img", "src")
+ // Rum & Co uses pagespeed
+ image_url_noisy := e.ChildAttr("img", "src")
+
+ if strings.Contains(image_url_noisy, "pagespeed") {
+ r_pagespeed, err := regexp.Compile(`jpg(\.pagespeed.+)$`)
+ if err != nil {
+ Fatal(err, "Rum & Co: Pagespeed regexp failed")
+ }
+ image_url_noisy_slice := r_pagespeed.FindStringSubmatch(image_url_noisy)
+ if len(image_url_noisy_slice) < 2 {
+ PrintlnOffer(W, "Rum & Co: (Pagespeed) Image URL not found")
+ return
+ }
+ image_url_noisy = strings.Replace(image_url_noisy, image_url_noisy_slice[1], "", 1)
+ }
+
+ W.Image_url = "https://www.rumundco.de/" + image_url_noisy
e.Request.Visit(W.Url)
W.Volume = get_volume(e)
if W.Volume == 0 {
+ PrintlnOffer(W, "Rum & Co: No Volume found")
return
}
- W.Abv = get_abv(e)
+ if "" == abv_noisy {
+ W.Abv = get_abv(e)
+ } else {
+ W.Abv, err = extract_abv(abv_noisy)
+ if err != nil {
+ Fatal(err, "Rum & Co: Base price: Extracting ABV failed")
+ }
+ }
W.Shop = shop.Id
W.Spirit_type = "Whisky"
diff --git a/crawler/shop_whic.go b/crawler/shop_whic.go
index b4f9562..e2bb6b9 100644
--- a/crawler/shop_whic.go
+++ b/crawler/shop_whic.go
@@ -1,7 +1,7 @@
package main
import (
- "log"
+ log "github.com/Sirupsen/logrus"
"regexp"
"strings"
diff --git a/crawler/shop_whiskyde.go b/crawler/shop_whiskyde.go
index 6038c11..e635546 100644
--- a/crawler/shop_whiskyde.go
+++ b/crawler/shop_whiskyde.go
@@ -1,7 +1,7 @@
package main
import (
- "log"
+ log "github.com/Sirupsen/logrus"
"strings"
"github.com/gocolly/colly"
diff --git a/crawler/shop_whiskysitenl.go b/crawler/shop_whiskysitenl.go
index 656cf18..a555123 100644
--- a/crawler/shop_whiskysitenl.go
+++ b/crawler/shop_whiskysitenl.go
@@ -1,7 +1,7 @@
package main
import (
- "log"
+ log "github.com/Sirupsen/logrus"
"regexp"
"strings"
diff --git a/crawler/shop_whiskyworld.go b/crawler/shop_whiskyworld.go
index c0fb7b6..9b968a3 100644
--- a/crawler/shop_whiskyworld.go
+++ b/crawler/shop_whiskyworld.go
@@ -1,7 +1,7 @@
package main
import (
- "log"
+ log "github.com/Sirupsen/logrus"
"strings"
"github.com/gocolly/colly"
diff --git a/crawler/shop_whiskyzone.go b/crawler/shop_whiskyzone.go
index 5809b7e..c07c14f 100644
--- a/crawler/shop_whiskyzone.go
+++ b/crawler/shop_whiskyzone.go
@@ -1,7 +1,6 @@
package main
import (
- "log"
"strings"
"github.com/gocolly/colly"
@@ -38,17 +37,20 @@ func ScrapeWhiskyzone(shop Shop) []Angebot {
e.Request.Visit(W.Url)
+ if "sold_out" == e.Request.Ctx.Get("sold_out") {
+ PrintlnOffer(W, "Whiskyzone: Sold out")
+ return
+ }
+
var err error
W.Discounted_price, err = convert_price(e.Request.Ctx.Get("discounted_price"))
if err != nil {
- log.Println("Discounted_price failed")
- log.Fatal(err)
+ Fatal(err, "Whiskyzone: Convert discounted price failed")
}
W.Original_price, err = convert_price(e.Request.Ctx.Get("original_price"))
if err != nil {
- log.Println("Original_price failed")
- log.Fatal(err)
+ Fatal(err, "Whiskyzone: Convert original price failed")
}
W.Volume = get_volume(e)
@@ -110,6 +112,16 @@ func ScrapeWhiskyzone(shop Shop) []Angebot {
e.Request.Ctx.Put("website", string(e.Response.Body))
})
+ // Product not found
+ c.OnHTML(".detail-error--headline", func(e *colly.HTMLElement) {
+ if e.Request.URL.String() == Shop_url {
+ return
+ }
+
+ e.Request.Ctx.Put("sold_out", "sold_out")
+
+ })
+
c.Visit(Shop_url)
return Whiskys
diff --git a/crawler/shops.go b/crawler/shops.go
index 9cad602..8cfd9f4 100644
--- a/crawler/shops.go
+++ b/crawler/shops.go
@@ -1,7 +1,7 @@
package main
import (
- "log"
+ log "github.com/Sirupsen/logrus"
)
func (app *App) insertShops() error {
@@ -112,7 +112,7 @@ func (app *App) getShops() ([]Shop, error) {
return []Shop{}, err
}
if app.Config.Debug {
- log.Println("Appending: " + shop.Name)
+ log.Println("Crawling: " + shop.Name)
}
Shops = append(Shops, shop)
diff --git a/crawler/utility.go b/crawler/utility.go
index c3daeb1..98587c9 100644
--- a/crawler/utility.go
+++ b/crawler/utility.go
@@ -1,7 +1,7 @@
package main
import (
- "log"
+ log "github.com/Sirupsen/logrus"
"regexp"
"strconv"
"strings"
@@ -21,56 +21,56 @@ func stringInSlice(a string, list []string) bool {
func detect_spirit_type(name string) string {
matched, err := regexp.MatchString(`(^|\s)Gin(\s|$)`, name)
if err != nil {
- log.Fatal(err)
+ Fatal(err, "Gin regex failed")
}
if matched {
return "Gin"
}
matched, err = regexp.MatchString(`(^|\s)Rh?um(\s|$)`, name)
if err != nil {
- log.Fatal(err)
+ Fatal(err, "Rum regex failed")
}
if matched {
return "Rum"
}
matched, err = regexp.MatchString(`(^|\s)[VW]odka(\s|$)`, name)
if err != nil {
- log.Fatal(err)
+ Fatal(err, "Wodka regex failed")
}
if matched {
return "Wodka"
}
matched, err = regexp.MatchString(`(^|\s)Whiske?y(\s|$)`, name)
if err != nil {
- log.Fatal(err)
+ Fatal(err, "Whisky regex failed")
}
if matched {
return "Whisky"
}
matched, err = regexp.MatchString(`(^|\s)Champagner(\s|$)`, name)
if err != nil {
- log.Fatal(err)
+ Fatal(err, "Champagner regex failed")
}
if matched {
return "Champagner"
}
matched, err = regexp.MatchString(`(^|\s)Cognac(\s|$)`, name)
if err != nil {
- log.Fatal(err)
+ Fatal(err, "Cognac regex failed")
}
if matched {
return "Cognac"
}
matched, err = regexp.MatchString(`(^|\s)Grappa(\s|$)`, name)
if err != nil {
- log.Fatal(err)
+ Fatal(err, "Grappa regex failed")
}
if matched {
return "Grappa"
}
matched, err = regexp.MatchString(`(^|\s)Likör(\s|$)`, name)
if err != nil {
- log.Fatal(err)
+ Fatal(err, "Likör regex failed")
}
if matched {
return "Likör"
@@ -82,18 +82,19 @@ func detect_spirit_type(name string) string {
func extract_volume(volume string) (float32, error) {
r_liter, err := regexp.Compile(`[0-9]+([,.][0-9]+)?( )?[lL](iter)?`)
if err != nil {
- log.Fatal(err)
+ Fatal(err, "Extract volume regex failed")
}
volume_noisy := r_liter.FindString(volume)
r_liter2, err := regexp.Compile(`[0-9]+([,.][0-9]+)?`)
if err != nil {
- log.Fatal(err)
+ Fatal(err, "2nd extract volume regex failed")
}
volume_noisy = r_liter2.FindString(volume_noisy)
volume_noisy = strings.Replace(volume_noisy, ",", ".", 1)
volume64, err := strconv.ParseFloat(volume_noisy, 32)
if err != nil {
+ Println(err, "Parsing volume to float failed")
return 0, err
}
@@ -114,12 +115,13 @@ func extract_abv(abv_noisy string) (float32, error) {
abv_noisy = strings.TrimSpace(abv_noisy)
r_abv, err := regexp.Compile(`[0-9]+([,.][0-9]+)?`)
if err != nil {
- log.Fatal(err)
+ Fatal(err, "Extract abv regex failed")
}
abv_noisy = r_abv.FindString(abv_noisy)
abv64, err := strconv.ParseFloat(abv_noisy, 32)
if err != nil {
+ Println(err, "Parsing abv to float failed")
return 0, err
}
@@ -135,6 +137,7 @@ func get_volume(e *colly.HTMLElement) float32 {
matched, err := regexp.MatchString(`[lL](iter)?`, volume_noisy)
if err != nil {
+ Fatal(err, "Get volume regex failed")
log.Fatal(err)
}
if !matched {
@@ -145,7 +148,7 @@ func get_volume(e *colly.HTMLElement) float32 {
volume, err := extract_volume(volume_noisy)
if err != nil {
log.Println("get_volume: " + volume_noisy)
- log.Fatal(err)
+ Fatal(err, "Get Volume: Extract Volume failed")
}
return volume
@@ -166,7 +169,7 @@ func get_abv(e *colly.HTMLElement) float32 {
abv, err := extract_abv(abv_noisy)
if err != nil {
log.Println("get_abv: " + abv_noisy)
- log.Fatal(err)
+ Fatal(err, "Get ABV: Extract ABV failed")
}
return abv
@@ -186,7 +189,7 @@ func get_base_price(e *colly.HTMLElement) int {
base_price, err := sanitize_base_price(base_price_noisy)
if err != nil {
log.Println("get_base_price: " + base_price_noisy)
- log.Fatal(err)
+ Fatal(err, "Get base price: sanitize base price failed")
}
return base_price