diff options
Diffstat (limited to 'crawler')
| -rw-r--r-- | crawler/main.go | 2 | ||||
| -rw-r--r-- | crawler/scrape.go | 10 |
2 files changed, 6 insertions, 6 deletions
diff --git a/crawler/main.go b/crawler/main.go index 54c351f..85206e9 100644 --- a/crawler/main.go +++ b/crawler/main.go @@ -94,7 +94,7 @@ func main() { return } - app.ScrapeHTML(shops) + app.Scrape(shops) // short url err = app.post_process() diff --git a/crawler/scrape.go b/crawler/scrape.go index 927bb48..f46b651 100644 --- a/crawler/scrape.go +++ b/crawler/scrape.go @@ -7,14 +7,14 @@ import ( log "github.com/sirupsen/logrus" ) -func (app *App) ScrapeHTML(shops []Shop) { +func (app *App) Scrape(shops []Shop) { wait := make(chan bool) count := 0 for _, shop := range shops { - go app.Scrape(shop, wait) + go app.ScrapeShop(shop, wait) count++ } @@ -25,13 +25,13 @@ func (app *App) ScrapeHTML(shops []Shop) { } } -func (app *App) Scrape(shop Shop, wait chan bool) { +func (app *App) ScrapeShop(shop Shop, wait chan bool) { var W []Angebot var err error // retry on error for i := 1; i < 4; i++ { - W = app.ScrapeShop(shop) + W = app.ScrapeHTML(shop) W = sanitize_offer(W, shop, i) @@ -59,7 +59,7 @@ func (app *App) Scrape(shop Shop, wait chan bool) { wait <- true } -func (app *App) ScrapeShop(shop Shop) []Angebot { +func (app *App) ScrapeHTML(shop Shop) []Angebot { switch shop.Name { case "Bottleworld": |