Crawler saves whole webpage.

5 files changed, 20 insertions, 0 deletions

diff --git a/crawler/shop_bottleworld.go b/crawler/shop_bottleworld.go
index 3eb151d..06082d4 100644
--- a/crawler/shop_bottleworld.go
+++ b/crawler/shop_bottleworld.go
@@ -61,6 +61,7 @@ func ScrapeBottleWord(shop Shop) []Angebot {
 		W.Abv = get_abv(e)
 
 		W.Spirit_type = e.Request.Ctx.Get("spirit_type")
+		W.Website = e.Request.Ctx.Get("website")
 
 		Whiskys = append(Whiskys, W)
 	})
@@ -78,6 +79,8 @@ func ScrapeBottleWord(shop Shop) []Angebot {
 				e.Request.Ctx.Put("abv", td_str)
 			}
 		})
+
+		e.Request.Ctx.Put("website", string(e.Response.Body))
 	})
 
 	c.OnHTML(".short-description", func(e *colly.HTMLElement) {
diff --git a/crawler/shop_mcwhisky.go b/crawler/shop_mcwhisky.go
index b44e892..2d4c0db 100644
--- a/crawler/shop_mcwhisky.go
+++ b/crawler/shop_mcwhisky.go
@@ -65,6 +65,8 @@ func ScrapeMCWhisky(shop Shop) []Angebot {
 		W.Shop = shop.Id
 		W.Spirit_type = "Whisky"
 
+		W.Website = e.Request.Ctx.Get("website")
+
 		Whiskys = append(Whiskys, W)
 	})
 
@@ -84,6 +86,7 @@ func ScrapeMCWhisky(shop Shop) []Angebot {
 		}
 
 		e.Request.Ctx.Put("volume", r_volume.FindString(text_noisy))
+		e.Request.Ctx.Put("website", string(e.Response.Body))
 
 	})
 
diff --git a/crawler/shop_rumundco.go b/crawler/shop_rumundco.go
index ef98e90..cd67c0f 100644
--- a/crawler/shop_rumundco.go
+++ b/crawler/shop_rumundco.go
@@ -80,6 +80,7 @@ func ScrapeRumundCo(shop Shop) []Angebot {
 
 		W.Shop = shop.Id
 		W.Spirit_type = "Whisky"
+		W.Website = e.Request.Ctx.Get("website")
 
 		Whiskys = append(Whiskys, W)
 	})
@@ -98,6 +99,7 @@ func ScrapeRumundCo(shop Shop) []Angebot {
 				e.Request.Ctx.Put("abv", e.ChildText("a"))
 			}
 		})
+		e.Request.Ctx.Put("website", string(e.Response.Body))
 	})
 
 	c.Visit(Shop_url)
diff --git a/crawler/shop_whic.go b/crawler/shop_whic.go
index 6025050..b4f9562 100644
--- a/crawler/shop_whic.go
+++ b/crawler/shop_whic.go
@@ -71,6 +71,7 @@ func ScrapeWhic(shop Shop) []Angebot {
 
 		W.Shop = shop.Id
 		W.Spirit_type = "Whisky"
+		W.Website = e.Request.Ctx.Get("website")
 
 		Whiskys = append(Whiskys, W)
 	})
@@ -99,6 +100,7 @@ func ScrapeWhic(shop Shop) []Angebot {
 		}
 
 		e.Request.Ctx.Put("abv", abv[1]+"%")
+		e.Request.Ctx.Put("website", string(e.Response.Body))
 	})
 
 	c.Visit(Shop_url)
diff --git a/crawler/shop_whiskyde.go b/crawler/shop_whiskyde.go
index b450c86..77e4bcb 100644
--- a/crawler/shop_whiskyde.go
+++ b/crawler/shop_whiskyde.go
@@ -81,9 +81,19 @@ func ScrapeWhiskyde(shop Shop) []Angebot {
 			log.Fatal(err)
 		}
 
+		e.Request.Visit(W.Url)
+		W.Website = e.Request.Ctx.Get("website")
+
 		Whiskys = append(Whiskys, W)
 	})
 
+	c.OnHTML("body", func(e *colly.HTMLElement) {
+		if e.Request.URL.String() == Shop_url {
+			return
+		}
+		e.Request.Ctx.Put("website", string(e.Response.Body))
+	})
+
 	c.Visit(Shop_url)
 
 	return Whiskys