summaryrefslogtreecommitdiff
path: root/crawler/scrape.go
diff options
context:
space:
mode:
authorMaximilian Möhring2019-05-15 16:58:56 +0200
committerMaximilian Möhring2019-05-15 16:58:56 +0200
commitc9afd4e8afe6a719b5d930c9baa2699442849fd8 (patch)
tree3eb127ab39e8cacabef7c82a264f18ce4b8ee321 /crawler/scrape.go
parent428ed1c35be88fbfedc1569d6e9692778c89ff25 (diff)
downloadalkobote-c9afd4e8afe6a719b5d930c9baa2699442849fd8.tar.gz
Improves structured logging. (crawler)
Diffstat (limited to 'crawler/scrape.go')
-rw-r--r--crawler/scrape.go9
1 files changed, 5 insertions, 4 deletions
diff --git a/crawler/scrape.go b/crawler/scrape.go
index f46b651..9d0d0f2 100644
--- a/crawler/scrape.go
+++ b/crawler/scrape.go
@@ -4,7 +4,6 @@ import (
"time"
"github.com/gocolly/colly"
- log "github.com/sirupsen/logrus"
)
func (app *App) Scrape(shops []Shop) {
@@ -49,11 +48,13 @@ func (app *App) ScrapeShop(shop Shop, wait chan bool) {
err = app.save_offer(W)
if err != nil {
- Warn(err, "Saving offers failed. Shop: "+shop.Name)
+ shop.error_msg = err.Error()
+ shop.Warn("Saving offers failed.")
}
err = app.remove_expired(W, shop)
if err != nil {
- Warn(err, "Removing expired offers failed. Shop: "+shop.Name)
+ shop.error_msg = err.Error()
+ shop.Warn("Removing expired offers failed.")
}
wait <- true
@@ -81,7 +82,7 @@ func (app *App) ScrapeHTML(shop Shop) []Angebot {
case "Drankdozijn":
return app.ScrapeDrankdozijn(shop)
default:
- log.Println(shop.Name + ": No Crawler")
+ shop.Warn("No Crawler")
}
return []Angebot{}