summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--crawler/shop_bottleworld.go4
-rw-r--r--crawler/shop_mcwhisky.go6
-rw-r--r--crawler/shop_rumundco.go6
-rw-r--r--crawler/shop_whic.go5
-rw-r--r--crawler/shop_whiskyde.go7
-rw-r--r--crawler/shop_whiskysitenl.go4
-rw-r--r--crawler/shop_whiskyworld.go6
7 files changed, 22 insertions, 16 deletions
diff --git a/crawler/shop_bottleworld.go b/crawler/shop_bottleworld.go
index 720f2c8..0057bff 100644
--- a/crawler/shop_bottleworld.go
+++ b/crawler/shop_bottleworld.go
@@ -20,6 +20,8 @@ func (app *App) ScrapeBottleWord(shop Shop) []Angebot {
c.OnHTML("li.item", func(e *colly.HTMLElement) {
W := Angebot{}
+ W.Shop = shop.Id
+
whisky_name := e.ChildText("h2 > a")
var err error
@@ -63,8 +65,6 @@ func (app *App) ScrapeBottleWord(shop Shop) []Angebot {
e.Request.Visit(W.Url)
- W.Shop = shop.Id
-
var ctx string
W.Volume, ctx = get_volume(e)
if W.Volume == 0 {
diff --git a/crawler/shop_mcwhisky.go b/crawler/shop_mcwhisky.go
index 2a188c3..87d822e 100644
--- a/crawler/shop_mcwhisky.go
+++ b/crawler/shop_mcwhisky.go
@@ -24,6 +24,9 @@ func (app *App) ScrapeMCWhisky(shop Shop) []Angebot {
W := Angebot{}
+ W.Shop = shop.Id
+ W.Spirit_type = "Whisky"
+
whisky_name := e.ChildAttr("a", "title")
whisky_url := e.ChildAttr("a", "href")
W.Name = whisky_name
@@ -96,9 +99,6 @@ func (app *App) ScrapeMCWhisky(shop Shop) []Angebot {
return
}
- W.Shop = shop.Id
- W.Spirit_type = "Whisky"
-
W.Website = e.Request.Ctx.Get("website")
Whiskys = append(Whiskys, W)
diff --git a/crawler/shop_rumundco.go b/crawler/shop_rumundco.go
index 58de518..30c8765 100644
--- a/crawler/shop_rumundco.go
+++ b/crawler/shop_rumundco.go
@@ -27,6 +27,9 @@ func (app *App) ScrapeRumundCo(shop Shop) []Angebot {
W := Angebot{}
+ W.Shop = shop.Id
+ W.Spirit_type = "Whisky"
+
whisky_name := strings.TrimPrefix(e.ChildAttr("img", "alt"), "Restposten: ")
whisky_url := "https://www.rumundco.de/" + e.ChildAttr("a", "href")
@@ -136,9 +139,6 @@ func (app *App) ScrapeRumundCo(shop Shop) []Angebot {
WarnOffer(W, "Rum & Co: Abv is zero")
return
}
-
- W.Shop = shop.Id
- W.Spirit_type = "Whisky"
W.Website = e.Request.Ctx.Get("website")
Whiskys = append(Whiskys, W)
diff --git a/crawler/shop_whic.go b/crawler/shop_whic.go
index 0e7cdf9..b5ff1af 100644
--- a/crawler/shop_whic.go
+++ b/crawler/shop_whic.go
@@ -24,6 +24,9 @@ func (app *App) ScrapeWhic(shop Shop) []Angebot {
W := Angebot{}
+ W.Shop = shop.Id
+ W.Spirit_type = "Whisky"
+
whisky_name := e.ChildAttr("a", "title")
whisky_url := e.ChildAttr("a", "href")
@@ -93,8 +96,6 @@ func (app *App) ScrapeWhic(shop Shop) []Angebot {
return
}
- W.Shop = shop.Id
- W.Spirit_type = "Whisky"
W.Website = e.Request.Ctx.Get("website")
Whiskys = append(Whiskys, W)
diff --git a/crawler/shop_whiskyde.go b/crawler/shop_whiskyde.go
index ffbbe08..7ad7fd7 100644
--- a/crawler/shop_whiskyde.go
+++ b/crawler/shop_whiskyde.go
@@ -23,6 +23,10 @@ func (app *App) ScrapeWhiskyde(shop Shop) []Angebot {
}
W := Angebot{}
+
+ W.Shop = shop.Id
+ W.Spirit_type = "Whisky"
+
whisky_name := e.ChildAttr("a", "title")
W.Name = whisky_name
whisky_url := strings.Replace(e.ChildAttr("a", "href"), "?&searchorigin=2", "", 1)
@@ -95,9 +99,6 @@ func (app *App) ScrapeWhiskyde(shop Shop) []Angebot {
return
}
- W.Shop = shop.Id
- W.Spirit_type = "Whisky"
-
W.Base_price, err = convert_price(e.ChildText(".article-unitprice-default"))
if err != nil {
W.error_msg = err.Error()
diff --git a/crawler/shop_whiskysitenl.go b/crawler/shop_whiskysitenl.go
index d69ec68..e410445 100644
--- a/crawler/shop_whiskysitenl.go
+++ b/crawler/shop_whiskysitenl.go
@@ -8,6 +8,7 @@ import (
)
func (app *App) ScrapeWhiskysitenl(shop Shop) []Angebot {
+
Whiskys := []Angebot{}
Shop_url := "https://www.whiskysite.nl/en/specials/?limit=100"
@@ -94,6 +95,8 @@ func (app *App) ScrapeWhiskysitenl(shop Shop) []Angebot {
// calculate base price, volume is never zero
W.Base_price = int(RoundToEven(float64(W.Discounted_price) / float64(W.Volume)))
+ W.Website = e.Request.Ctx.Get("website")
+
Whiskys = append(Whiskys, W)
})
@@ -135,6 +138,7 @@ func (app *App) ScrapeWhiskysitenl(shop Shop) []Angebot {
}
e.Request.Ctx.Put("abv", abv_noisy)
+ e.Request.Ctx.Put("website", string(e.Response.Body))
})
c.Visit(Shop_url)
diff --git a/crawler/shop_whiskyworld.go b/crawler/shop_whiskyworld.go
index def22c6..6306f6f 100644
--- a/crawler/shop_whiskyworld.go
+++ b/crawler/shop_whiskyworld.go
@@ -28,6 +28,9 @@ func (app *App) ScrapeWhiskyworld(shop Shop) []Angebot {
W := Angebot{}
+ W.Shop = shop.Id
+ W.Spirit_type = "Whisky"
+
whisky_name_part1 := e.ChildText("h3")
whisky_name_part2 := e.ChildText(".item-description")
@@ -105,9 +108,6 @@ func (app *App) ScrapeWhiskyworld(shop Shop) []Angebot {
W.Image_url = "https:" + e.ChildAttr("img", "src")
- W.Shop = shop.Id
- W.Spirit_type = "Whisky"
-
e.Request.Visit(W.Url)
W.Website = e.Request.Ctx.Get("website")