summaryrefslogtreecommitdiff
path: root/crawler/scrape.go
diff options
context:
space:
mode:
authorhorus_arch2018-02-17 13:51:35 +0100
committerhorus_arch2018-02-17 13:51:35 +0100
commitbcdea2f8e95f5305625a773223829478c8c13bed (patch)
treeefac40b03131b4f9e43de848920695ee785a0a3f /crawler/scrape.go
parent9ebf51364773dae6db4c0c47d77710c9f1a37b51 (diff)
downloadalkobote-bcdea2f8e95f5305625a773223829478c8c13bed.tar.gz
Introduces context on errors. (crawler)
Diffstat (limited to 'crawler/scrape.go')
-rw-r--r--crawler/scrape.go24
1 files changed, 12 insertions, 12 deletions
diff --git a/crawler/scrape.go b/crawler/scrape.go
index f6ad80b..ae63e5c 100644
--- a/crawler/scrape.go
+++ b/crawler/scrape.go
@@ -28,41 +28,41 @@ func (app *App) Scrape(shop Shop, wait chan bool) {
var W []Angebot
var err error
- W = ScrapeShop(shop)
+ W = app.ScrapeShop(shop)
W = sanitize_offer(W, shop)
err = app.save_offer(W)
if err != nil {
- Fatal(err, "Saving offers failed")
+ Warn(err, "Saving offers failed")
}
err = app.remove_expired(W, shop)
if err != nil {
- Fatal(err, "Removing expired offers failed")
+ Warn(err, "Removing expired offers failed")
}
wait <- true
}
-func ScrapeShop(shop Shop) []Angebot {
+func (app *App) ScrapeShop(shop Shop) []Angebot {
switch shop.Name {
case "Bottleworld":
- return ScrapeBottleWord(shop)
+ return app.ScrapeBottleWord(shop)
case "MC Whisky":
- return ScrapeMCWhisky(shop)
+ return app.ScrapeMCWhisky(shop)
case "Rum & Co":
- return ScrapeRumundCo(shop)
+ return app.ScrapeRumundCo(shop)
case "Whic":
- return ScrapeWhic(shop)
+ return app.ScrapeWhic(shop)
case "Whisky.de":
- return ScrapeWhiskyde(shop)
+ return app.ScrapeWhiskyde(shop)
//case "Whiskysite.nl":
- // return ScrapeWhiskysitenl(shop)
+ // return app.ScrapeWhiskysitenl(shop)
case "Whisky World":
- return ScrapeWhiskyworld(shop)
+ return app.ScrapeWhiskyworld(shop)
case "Whiskyzone":
- return ScrapeWhiskyzone(shop)
+ return app.ScrapeWhiskyzone(shop)
default:
log.Println(shop.Name + ": No Crawler")
}