Improves structured logging. (crawler)

14 files changed, 97 insertions, 53 deletions

diff --git a/crawler/log.go b/crawler/log.go
index 111df53..e9481cb 100644
--- a/crawler/log.go
+++ b/crawler/log.go
@@ -4,6 +4,32 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
+func (offer Angebot) getFields() log.Fields {
+	return log.Fields{
+		"Name":             offer.Name,
+		"Shop":             offer.Shop,
+		"ABV":              offer.Abv,
+		"Volume":           offer.Volume,
+		"Url":              offer.Url,
+		"Original Price":   offer.Original_price,
+		"Discounted Price": offer.Discounted_price,
+		"Base Price":       offer.Base_price,
+		"Image_url":        offer.Image_url,
+		"Spirit Type":      offer.Spirit_type,
+		"Valid Until":      offer.Valid_until,
+		"Error Msg":        offer.error_msg,
+		"Error Ctx":        offer.error_ctx,
+	}
+}
+
+func (shop Shop) getFields() log.Fields {
+	return log.Fields{
+		"Name":      shop.Name,
+		"Error Msg": shop.error_msg,
+		"Error Ctx": shop.error_ctx,
+	}
+}
+
 func Fatal(err error, msg string) {
 	if err != nil {
 		log.WithFields(
@@ -15,9 +41,33 @@ func Fatal(err error, msg string) {
 		log.Fatal(msg)
 	}
 }
+
 func (offer Angebot) Fatal(msg string) {
 	log.WithFields(offer.getFields()).Fatal(msg)
 }
+func (shop Shop) Fatal(msg string) {
+	log.WithFields(shop.getFields()).Fatal(msg)
+}
+
+func Warn(err error, msg string) {
+	if err != nil {
+		log.WithFields(
+			log.Fields{
+				"error": err.Error(),
+			},
+		).Warn(msg)
+	} else {
+		log.Warn(msg)
+	}
+}
+
+func (offer Angebot) Warn(msg string) {
+	log.WithFields(offer.getFields()).Warn(msg)
+}
+
+func (shop Shop) Warn(msg string) {
+	log.WithFields(shop.getFields()).Warn(msg)
+}
 
 func Println(err error, msg string) {
 	if err != nil {
@@ -32,10 +82,13 @@ func Println(err error, msg string) {
 }
 
 func (offer Angebot) Println(msg string) {
-
 	log.WithFields(offer.getFields()).Println(msg)
 }
 
+func (shop Shop) Println(msg string) {
+	log.WithFields(shop.getFields()).Println(msg)
+}
+
 func Debug(err error, msg string) {
 	if err != nil {
 		log.WithFields(
@@ -49,10 +102,13 @@ func Debug(err error, msg string) {
 }
 
 func (offer Angebot) Debug(msg string) {
-
 	log.WithFields(offer.getFields()).Debug(msg)
 }
 
+func (shop Shop) Debug(msg string) {
+	log.WithFields(shop.getFields()).Debug(msg)
+}
+
 func Trace(err error, msg string) {
 	if err != nil {
 		log.WithFields(
@@ -66,41 +122,9 @@ func Trace(err error, msg string) {
 }
 
 func (offer Angebot) Trace(msg string) {
-
 	log.WithFields(offer.getFields()).Trace(msg)
 }
 
-func (offer Angebot) Warn(msg string) {
-
-	log.WithFields(offer.getFields()).Warn(msg)
-}
-
-func (offer Angebot) getFields() log.Fields {
-	return log.Fields{
-		"Name":             offer.Name,
-		"Shop":             offer.Shop,
-		"ABV":              offer.Abv,
-		"Volume":           offer.Volume,
-		"Url":              offer.Url,
-		"Original Price":   offer.Original_price,
-		"Discounted Price": offer.Discounted_price,
-		"Base Price":       offer.Base_price,
-		"Image_url":        offer.Image_url,
-		"Spirit Type":      offer.Spirit_type,
-		"Valid Until":      offer.Valid_until,
-		"Error_msg":        offer.error_msg,
-		"Error_ctx":        offer.error_ctx,
-	}
-}
-
-func Warn(err error, msg string) {
-	if err != nil {
-		log.WithFields(
-			log.Fields{
-				"error": err.Error(),
-			},
-		).Warn(msg)
-	} else {
-		log.Warn(msg)
-	}
+func (shop Shop) Trace(msg string) {
+	log.WithFields(shop.getFields()).Trace(msg)
 }
diff --git a/crawler/sanitize.go b/crawler/sanitize.go
index 8eb9444..031dd45 100644
--- a/crawler/sanitize.go
+++ b/crawler/sanitize.go
@@ -73,7 +73,7 @@ func sanitize_offer(angebote []Angebot, shop Shop, try int) []Angebot {
 	}
 
 	if len(W) < 1 {
-		log.Warn(fmt.Sprintf(`Sanitizer: No results for shop: '%s' (%d.) try`, shop.Name, try))
+		shop.Warn(fmt.Sprintf(`Sanitizer: No results (%d. try)`, try))
 	}
 
 	return W
diff --git a/crawler/scrape.go b/crawler/scrape.go
index f46b651..9d0d0f2 100644
--- a/crawler/scrape.go
+++ b/crawler/scrape.go
@@ -4,7 +4,6 @@ import (
 	"time"
 
 	"github.com/gocolly/colly"
-	log "github.com/sirupsen/logrus"
 )
 
 func (app *App) Scrape(shops []Shop) {
@@ -49,11 +48,13 @@ func (app *App) ScrapeShop(shop Shop, wait chan bool) {
 
 	err = app.save_offer(W)
 	if err != nil {
-		Warn(err, "Saving offers failed. Shop: "+shop.Name)
+		shop.error_msg = err.Error()
+		shop.Warn("Saving offers failed.")
 	}
 	err = app.remove_expired(W, shop)
 	if err != nil {
-		Warn(err, "Removing expired offers failed. Shop: "+shop.Name)
+		shop.error_msg = err.Error()
+		shop.Warn("Removing expired offers failed.")
 	}
 
 	wait <- true
@@ -81,7 +82,7 @@ func (app *App) ScrapeHTML(shop Shop) []Angebot {
 	case "Drankdozijn":
 		return app.ScrapeDrankdozijn(shop)
 	default:
-		log.Println(shop.Name + ": No Crawler")
+		shop.Warn("No Crawler")
 	}
 
 	return []Angebot{}
diff --git a/crawler/shop_bottleworld.go b/crawler/shop_bottleworld.go
index ce49ae1..0f1a301 100644
--- a/crawler/shop_bottleworld.go
+++ b/crawler/shop_bottleworld.go
@@ -121,7 +121,9 @@ func (app *App) ScrapeBottleWord(shop Shop) []Angebot {
 
 	err := c.Visit(Shop_url)
 	if err != nil {
-		Warn(nil, shop.Name+": "+err.Error())
+		shop.error_msg = err.Error()
+		shop.error_ctx = Shop_url
+		shop.Warn("Crawling failed")
 	}
 
 	return Whiskys
diff --git a/crawler/shop_drankdozijn.go b/crawler/shop_drankdozijn.go
index 28f79b2..fdfc3d8 100644
--- a/crawler/shop_drankdozijn.go
+++ b/crawler/shop_drankdozijn.go
@@ -248,7 +248,9 @@ func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot {
 
 		err = c.Visit(W.Url)
 		if err != nil {
-			Warn(nil, shop.Name+": Error (Visit): "+err.Error())
+			shop.error_msg = err.Error()
+			shop.error_ctx = W.Url
+			shop.Warn("Crawling failed")
 		} else {
 			//log.Println("Visit " + W.Url)
 		}
diff --git a/crawler/shop_mcwhisky.go b/crawler/shop_mcwhisky.go
index 027aeee..89c2a7b 100644
--- a/crawler/shop_mcwhisky.go
+++ b/crawler/shop_mcwhisky.go
@@ -132,7 +132,9 @@ func (app *App) ScrapeMCWhisky(shop Shop) []Angebot {
 
 	err := c.Visit(Shop_url)
 	if err != nil {
-		Warn(nil, shop.Name+": "+err.Error())
+		shop.error_msg = err.Error()
+		shop.error_ctx = Shop_url
+		shop.Warn("Crawling failed")
 	}
 
 	return Whiskys
diff --git a/crawler/shop_rumundco.go b/crawler/shop_rumundco.go
index 3b7d08a..c9a61a2 100644
--- a/crawler/shop_rumundco.go
+++ b/crawler/shop_rumundco.go
@@ -199,7 +199,9 @@ func (app *App) ScrapeRumundCo(shop Shop) []Angebot {
 	for _, url := range Shop_urls {
 		err := c.Visit(url)
 		if err != nil {
-			Warn(nil, shop.Name+": "+err.Error())
+			shop.error_msg = err.Error()
+			shop.error_ctx = url
+			shop.Warn("Crawling failed")
 		}
 	}
 
diff --git a/crawler/shop_whic.go b/crawler/shop_whic.go
index b21c042..0df7caf 100644
--- a/crawler/shop_whic.go
+++ b/crawler/shop_whic.go
@@ -129,7 +129,9 @@ func (app *App) ScrapeWhic(shop Shop) []Angebot {
 
 	err := c.Visit(Shop_url)
 	if err != nil {
-		Warn(nil, shop.Name+": "+err.Error())
+		shop.error_msg = err.Error()
+		shop.error_ctx = Shop_url
+		shop.Warn("Crawling failed")
 	}
 
 	return Whiskys
diff --git a/crawler/shop_whiskyde.go b/crawler/shop_whiskyde.go
index d06f522..1088d40 100644
--- a/crawler/shop_whiskyde.go
+++ b/crawler/shop_whiskyde.go
@@ -120,7 +120,9 @@ func (app *App) ScrapeWhiskyde(shop Shop) []Angebot {
 
 	err := c.Visit(Shop_url)
 	if err != nil {
-		Warn(nil, shop.Name+": "+err.Error())
+		shop.error_msg = err.Error()
+		shop.error_ctx = Shop_url
+		shop.Warn("Crawling failed")
 	}
 
 	return Whiskys
diff --git a/crawler/shop_whiskysitenl.go b/crawler/shop_whiskysitenl.go
index c9a50b0..18b4a7b 100644
--- a/crawler/shop_whiskysitenl.go
+++ b/crawler/shop_whiskysitenl.go
@@ -140,7 +140,9 @@ func (app *App) ScrapeWhiskysitenl(shop Shop) []Angebot {
 
 	err := c.Visit(Shop_url)
 	if err != nil {
-		Warn(nil, shop.Name+": "+err.Error())
+		shop.error_msg = err.Error()
+		shop.error_ctx = Shop_url
+		shop.Warn("Crawling failed")
 	}
 
 	return Whiskys
diff --git a/crawler/shop_whiskyworld.go b/crawler/shop_whiskyworld.go
index 78e7244..bdd07bf 100644
--- a/crawler/shop_whiskyworld.go
+++ b/crawler/shop_whiskyworld.go
@@ -125,7 +125,9 @@ func (app *App) ScrapeWhiskyworld(shop Shop) []Angebot {
 	for _, url := range Shop_urls {
 		err := c.Visit(url)
 		if err != nil {
-			Warn(nil, shop.Name+": "+err.Error())
+			shop.error_msg = err.Error()
+			shop.error_ctx = url
+			shop.Warn("Crawling failed")
 		}
 	}
 
diff --git a/crawler/shop_whiskyzone.go b/crawler/shop_whiskyzone.go
index ffb124d..fa9d764 100644
--- a/crawler/shop_whiskyzone.go
+++ b/crawler/shop_whiskyzone.go
@@ -156,7 +156,9 @@ func (app *App) ScrapeWhiskyzone(shop Shop) []Angebot {
 
 	err := c.Visit(Shop_url)
 	if err != nil {
-		Warn(nil, shop.Name+": "+err.Error())
+		shop.error_msg = err.Error()
+		shop.error_ctx = Shop_url
+		shop.Warn("Crawling failed")
 	}
 
 	return Whiskys
diff --git a/crawler/shops.go b/crawler/shops.go
index 95d8c99..656cd1f 100644
--- a/crawler/shops.go
+++ b/crawler/shops.go
@@ -2,8 +2,6 @@ package main
 
 import (
 	"strings"
-
-	log "github.com/sirupsen/logrus"
 )
 
 func (app *App) insertShops() error {
@@ -134,7 +132,7 @@ func (app *App) getShops() ([]Shop, error) {
 		if err != nil {
 			return []Shop{}, err
 		}
-		log.Debug("Crawling: " + shop.Name)
+		shop.Debug("Crawling: " + shop.Name)
 
 		Shops = append(Shops, shop)
 	}
diff --git a/crawler/struct.go b/crawler/struct.go
index 1965c2e..1b0bfc3 100644
--- a/crawler/struct.go
+++ b/crawler/struct.go
@@ -29,4 +29,7 @@ type Shop struct {
 	Logo_url       string
 	Shipping_costs int
 	Free_shipping  string
+
+	error_msg string
+	error_ctx string
 }