diff options
| -rw-r--r-- | crawler/scrape.go | 4 | ||||
| -rw-r--r-- | crawler/shop_bottleworld.go | 5 |
2 files changed, 0 insertions, 9 deletions
diff --git a/crawler/scrape.go b/crawler/scrape.go index f9e758d..de79813 100644 --- a/crawler/scrape.go +++ b/crawler/scrape.go @@ -29,10 +29,6 @@ func (app *App) Scrape(shop Shop, wait chan bool) { var W []Angebot var err error - if err != nil { - Fatal(err, "scrape.go: Starting transaction failed. Shop: "+shop.Name) - } - // retry on error for i := 1; i < 4; i++ { W = app.ScrapeShop(shop) diff --git a/crawler/shop_bottleworld.go b/crawler/shop_bottleworld.go index d679b43..8722211 100644 --- a/crawler/shop_bottleworld.go +++ b/crawler/shop_bottleworld.go @@ -6,8 +6,6 @@ import ( // "github.com/PuerkitoBio/goquery" "github.com/gocolly/colly" - "log" - "time" ) func (app *App) ScrapeBottleWord(shop Shop) []Angebot { @@ -16,9 +14,6 @@ func (app *App) ScrapeBottleWord(shop Shop) []Angebot { c := app.customCollector([]string{"bottleworld.de", "www.bottleworld.de"}) - log.Println(c.IgnoreRobotsTxt) - log.Println(time.Duration(app.Config.Delay)) - c.OnHTML("li.item", func(e *colly.HTMLElement) { W := Angebot{} |