From ca5ac0bcb9206e81faab60cc8a8d6da697bdfdbe Mon Sep 17 00:00:00 2001
From: horus
Date: Mon, 12 Feb 2018 22:54:55 +0100
Subject: Crawler saves whole webpage.

---
 crawler/shop_whiskyde.go | 10 ++++++++++
 1 file changed, 10 insertions(+)

(limited to 'crawler/shop_whiskyde.go')

diff --git a/crawler/shop_whiskyde.go b/crawler/shop_whiskyde.go
index b450c86..77e4bcb 100644
--- a/crawler/shop_whiskyde.go
+++ b/crawler/shop_whiskyde.go
@@ -81,9 +81,19 @@ func ScrapeWhiskyde(shop Shop) []Angebot {
 			log.Fatal(err)
 		}
 
+		e.Request.Visit(W.Url)
+		W.Website = e.Request.Ctx.Get("website")
+
 		Whiskys = append(Whiskys, W)
 	})
 
+	c.OnHTML("body", func(e *colly.HTMLElement) {
+		if e.Request.URL.String() == Shop_url {
+			return
+		}
+		e.Request.Ctx.Put("website", string(e.Response.Body))
+	})
+
 	c.Visit(Shop_url)
 
 	return Whiskys
-- 
cgit v1.2.3