diff options
| -rw-r--r-- | crawler/shop_bottleworld.go | 3 | ||||
| -rw-r--r-- | crawler/shop_mcwhisky.go | 3 | ||||
| -rw-r--r-- | crawler/shop_rumundco.go | 2 | ||||
| -rw-r--r-- | crawler/shop_whic.go | 2 | ||||
| -rw-r--r-- | crawler/shop_whiskyde.go | 10 |
5 files changed, 20 insertions, 0 deletions
diff --git a/crawler/shop_bottleworld.go b/crawler/shop_bottleworld.go index 3eb151d..06082d4 100644 --- a/crawler/shop_bottleworld.go +++ b/crawler/shop_bottleworld.go @@ -61,6 +61,7 @@ func ScrapeBottleWord(shop Shop) []Angebot { W.Abv = get_abv(e) W.Spirit_type = e.Request.Ctx.Get("spirit_type") + W.Website = e.Request.Ctx.Get("website") Whiskys = append(Whiskys, W) }) @@ -78,6 +79,8 @@ func ScrapeBottleWord(shop Shop) []Angebot { e.Request.Ctx.Put("abv", td_str) } }) + + e.Request.Ctx.Put("website", string(e.Response.Body)) }) c.OnHTML(".short-description", func(e *colly.HTMLElement) { diff --git a/crawler/shop_mcwhisky.go b/crawler/shop_mcwhisky.go index b44e892..2d4c0db 100644 --- a/crawler/shop_mcwhisky.go +++ b/crawler/shop_mcwhisky.go @@ -65,6 +65,8 @@ func ScrapeMCWhisky(shop Shop) []Angebot { W.Shop = shop.Id W.Spirit_type = "Whisky" + W.Website = e.Request.Ctx.Get("website") + Whiskys = append(Whiskys, W) }) @@ -84,6 +86,7 @@ func ScrapeMCWhisky(shop Shop) []Angebot { } e.Request.Ctx.Put("volume", r_volume.FindString(text_noisy)) + e.Request.Ctx.Put("website", string(e.Response.Body)) }) diff --git a/crawler/shop_rumundco.go b/crawler/shop_rumundco.go index ef98e90..cd67c0f 100644 --- a/crawler/shop_rumundco.go +++ b/crawler/shop_rumundco.go @@ -80,6 +80,7 @@ func ScrapeRumundCo(shop Shop) []Angebot { W.Shop = shop.Id W.Spirit_type = "Whisky" + W.Website = e.Request.Ctx.Get("website") Whiskys = append(Whiskys, W) }) @@ -98,6 +99,7 @@ func ScrapeRumundCo(shop Shop) []Angebot { e.Request.Ctx.Put("abv", e.ChildText("a")) } }) + e.Request.Ctx.Put("website", string(e.Response.Body)) }) c.Visit(Shop_url) diff --git a/crawler/shop_whic.go b/crawler/shop_whic.go index 6025050..b4f9562 100644 --- a/crawler/shop_whic.go +++ b/crawler/shop_whic.go @@ -71,6 +71,7 @@ func ScrapeWhic(shop Shop) []Angebot { W.Shop = shop.Id W.Spirit_type = "Whisky" + W.Website = e.Request.Ctx.Get("website") Whiskys = append(Whiskys, W) }) @@ -99,6 +100,7 @@ func ScrapeWhic(shop Shop) []Angebot { } e.Request.Ctx.Put("abv", abv[1]+"%") + e.Request.Ctx.Put("website", string(e.Response.Body)) }) c.Visit(Shop_url) diff --git a/crawler/shop_whiskyde.go b/crawler/shop_whiskyde.go index b450c86..77e4bcb 100644 --- a/crawler/shop_whiskyde.go +++ b/crawler/shop_whiskyde.go @@ -81,9 +81,19 @@ func ScrapeWhiskyde(shop Shop) []Angebot { log.Fatal(err) } + e.Request.Visit(W.Url) + W.Website = e.Request.Ctx.Get("website") + Whiskys = append(Whiskys, W) }) + c.OnHTML("body", func(e *colly.HTMLElement) { + if e.Request.URL.String() == Shop_url { + return + } + e.Request.Ctx.Put("website", string(e.Response.Body)) + }) + c.Visit(Shop_url) return Whiskys |