diff options
Diffstat (limited to 'crawler/scrape.go')
| -rw-r--r-- | crawler/scrape.go | 24 |
1 files changed, 22 insertions, 2 deletions
diff --git a/crawler/scrape.go b/crawler/scrape.go index 055b00e..6ef9fcf 100644 --- a/crawler/scrape.go +++ b/crawler/scrape.go @@ -27,6 +27,12 @@ func (app *App) ScrapeHTML(shops []Shop) { func (app *App) Scrape(shop Shop, wait chan bool) { var W []Angebot var err error + txFailed := false + + app.Tx, err = app.DB.Beginx() + if err != nil { + Fatal(err, "scrape.go: Starting transaction failed. Shop: "+shop.Name) + } // retry on error for i := 0; i < 3; i++ { @@ -41,11 +47,25 @@ func (app *App) Scrape(shop Shop, wait chan bool) { err = app.save_offer(W) if err != nil { - Warn(err, "Saving offers failed") + txFailed = true + Warn(err, "Saving offers failed. Shop: "+shop.Name) } err = app.remove_expired(W, shop) if err != nil { - Warn(err, "Removing expired offers failed") + txFailed = true + Warn(err, "Removing expired offers failed. Shop: "+shop.Name) + } + + if txFailed { + err = app.Tx.Rollback() + if err != nil { + Fatal(err, "scrape.go: Rollback transaction failed") + } + } else { + err = app.Tx.Commit() + if err != nil { + Fatal(err, "scrape.go: Committing transaction failed") + } } wait <- true |