diff options
Diffstat (limited to 'crawler/scrape.go')
| -rw-r--r-- | crawler/scrape.go | 44 |
1 files changed, 22 insertions, 22 deletions
diff --git a/crawler/scrape.go b/crawler/scrape.go index 31b3618..6250896 100644 --- a/crawler/scrape.go +++ b/crawler/scrape.go @@ -6,32 +6,12 @@ import ( func (app *App) ScrapeHTML(shops []Shop) { - wait := make(chan int) + wait := make(chan bool) count := 0 for _, shop := range shops { - go func() { - var W []Angebot - var err error - - W = ScrapeShop(shop) - - W = sanitize_offer(W) - - err = app.save_offer(W) - if err != nil { - log.Fatal(err) - } - err = app.remove_expired(W) - if err != nil { - log.Fatal(err) - } - - wait <- 1 - - }() - + go app.Scrape(shop, wait) count++ } @@ -44,6 +24,26 @@ func (app *App) ScrapeHTML(shops []Shop) { } } +func (app *App) Scrape(shop Shop, wait chan bool) { + var W []Angebot + var err error + + W = ScrapeShop(shop) + + W = sanitize_offer(W) + + err = app.save_offer(W) + if err != nil { + log.Fatal(err) + } + err = app.remove_expired(W) + if err != nil { + log.Fatal(err) + } + + wait <- true +} + func ScrapeShop(shop Shop) []Angebot { switch shop.Name { |