Minor improvement in logging. (crawler)

7 files changed, 22 insertions, 16 deletions

diff --git a/crawler/shop_bottleworld.go b/crawler/shop_bottleworld.go
index 720f2c8..0057bff 100644
--- a/crawler/shop_bottleworld.go
+++ b/crawler/shop_bottleworld.go
@@ -20,6 +20,8 @@ func (app *App) ScrapeBottleWord(shop Shop) []Angebot {
 	c.OnHTML("li.item", func(e *colly.HTMLElement) {
 		W := Angebot{}
 
+		W.Shop = shop.Id
+
 		whisky_name := e.ChildText("h2 > a")
 
 		var err error
@@ -63,8 +65,6 @@ func (app *App) ScrapeBottleWord(shop Shop) []Angebot {
 
 		e.Request.Visit(W.Url)
 
-		W.Shop = shop.Id
-
 		var ctx string
 		W.Volume, ctx = get_volume(e)
 		if W.Volume == 0 {
diff --git a/crawler/shop_mcwhisky.go b/crawler/shop_mcwhisky.go
index 2a188c3..87d822e 100644
--- a/crawler/shop_mcwhisky.go
+++ b/crawler/shop_mcwhisky.go
@@ -24,6 +24,9 @@ func (app *App) ScrapeMCWhisky(shop Shop) []Angebot {
 
 		W := Angebot{}
 
+		W.Shop = shop.Id
+		W.Spirit_type = "Whisky"
+
 		whisky_name := e.ChildAttr("a", "title")
 		whisky_url := e.ChildAttr("a", "href")
 		W.Name = whisky_name
@@ -96,9 +99,6 @@ func (app *App) ScrapeMCWhisky(shop Shop) []Angebot {
 			return
 		}
 
-		W.Shop = shop.Id
-		W.Spirit_type = "Whisky"
-
 		W.Website = e.Request.Ctx.Get("website")
 
 		Whiskys = append(Whiskys, W)
diff --git a/crawler/shop_rumundco.go b/crawler/shop_rumundco.go
index 58de518..30c8765 100644
--- a/crawler/shop_rumundco.go
+++ b/crawler/shop_rumundco.go
@@ -27,6 +27,9 @@ func (app *App) ScrapeRumundCo(shop Shop) []Angebot {
 
 		W := Angebot{}
 
+		W.Shop = shop.Id
+		W.Spirit_type = "Whisky"
+
 		whisky_name := strings.TrimPrefix(e.ChildAttr("img", "alt"), "Restposten: ")
 		whisky_url := "https://www.rumundco.de/" + e.ChildAttr("a", "href")
 
@@ -136,9 +139,6 @@ func (app *App) ScrapeRumundCo(shop Shop) []Angebot {
 			WarnOffer(W, "Rum & Co: Abv is zero")
 			return
 		}
-
-		W.Shop = shop.Id
-		W.Spirit_type = "Whisky"
 		W.Website = e.Request.Ctx.Get("website")
 
 		Whiskys = append(Whiskys, W)
diff --git a/crawler/shop_whic.go b/crawler/shop_whic.go
index 0e7cdf9..b5ff1af 100644
--- a/crawler/shop_whic.go
+++ b/crawler/shop_whic.go
@@ -24,6 +24,9 @@ func (app *App) ScrapeWhic(shop Shop) []Angebot {
 
 		W := Angebot{}
 
+		W.Shop = shop.Id
+		W.Spirit_type = "Whisky"
+
 		whisky_name := e.ChildAttr("a", "title")
 		whisky_url := e.ChildAttr("a", "href")
 
@@ -93,8 +96,6 @@ func (app *App) ScrapeWhic(shop Shop) []Angebot {
 			return
 		}
 
-		W.Shop = shop.Id
-		W.Spirit_type = "Whisky"
 		W.Website = e.Request.Ctx.Get("website")
 
 		Whiskys = append(Whiskys, W)
diff --git a/crawler/shop_whiskyde.go b/crawler/shop_whiskyde.go
index ffbbe08..7ad7fd7 100644
--- a/crawler/shop_whiskyde.go
+++ b/crawler/shop_whiskyde.go
@@ -23,6 +23,10 @@ func (app *App) ScrapeWhiskyde(shop Shop) []Angebot {
 		}
 
 		W := Angebot{}
+
+		W.Shop = shop.Id
+		W.Spirit_type = "Whisky"
+
 		whisky_name := e.ChildAttr("a", "title")
 		W.Name = whisky_name
 		whisky_url := strings.Replace(e.ChildAttr("a", "href"), "?&searchorigin=2", "", 1)
@@ -95,9 +99,6 @@ func (app *App) ScrapeWhiskyde(shop Shop) []Angebot {
 			return
 		}
 
-		W.Shop = shop.Id
-		W.Spirit_type = "Whisky"
-
 		W.Base_price, err = convert_price(e.ChildText(".article-unitprice-default"))
 		if err != nil {
 			W.error_msg = err.Error()
diff --git a/crawler/shop_whiskysitenl.go b/crawler/shop_whiskysitenl.go
index d69ec68..e410445 100644
--- a/crawler/shop_whiskysitenl.go
+++ b/crawler/shop_whiskysitenl.go
@@ -8,6 +8,7 @@ import (
 )
 
 func (app *App) ScrapeWhiskysitenl(shop Shop) []Angebot {
+
 	Whiskys := []Angebot{}
 
 	Shop_url := "https://www.whiskysite.nl/en/specials/?limit=100"
@@ -94,6 +95,8 @@ func (app *App) ScrapeWhiskysitenl(shop Shop) []Angebot {
 		// calculate base price, volume is never zero
 		W.Base_price = int(RoundToEven(float64(W.Discounted_price) / float64(W.Volume)))
 
+		W.Website = e.Request.Ctx.Get("website")
+
 		Whiskys = append(Whiskys, W)
 	})
 
@@ -135,6 +138,7 @@ func (app *App) ScrapeWhiskysitenl(shop Shop) []Angebot {
 		}
 		e.Request.Ctx.Put("abv", abv_noisy)
 
+		e.Request.Ctx.Put("website", string(e.Response.Body))
 	})
 
 	c.Visit(Shop_url)
diff --git a/crawler/shop_whiskyworld.go b/crawler/shop_whiskyworld.go
index def22c6..6306f6f 100644
--- a/crawler/shop_whiskyworld.go
+++ b/crawler/shop_whiskyworld.go
@@ -28,6 +28,9 @@ func (app *App) ScrapeWhiskyworld(shop Shop) []Angebot {
 
 		W := Angebot{}
 
+		W.Shop = shop.Id
+		W.Spirit_type = "Whisky"
+
 		whisky_name_part1 := e.ChildText("h3")
 		whisky_name_part2 := e.ChildText(".item-description")
 
@@ -105,9 +108,6 @@ func (app *App) ScrapeWhiskyworld(shop Shop) []Angebot {
 
 		W.Image_url = "https:" + e.ChildAttr("img", "src")
 
-		W.Shop = shop.Id
-		W.Spirit_type = "Whisky"
-
 		e.Request.Visit(W.Url)
 		W.Website = e.Request.Ctx.Get("website")