summaryrefslogtreecommitdiff
path: root/crawler/scrape.go
diff options
context:
space:
mode:
authorhorus2018-02-20 13:05:01 +0100
committerhorus2018-02-20 13:05:01 +0100
commit4d3d10f634e872d0d4f27311c53f66680e574ad3 (patch)
treee962b026de5cd9b3d75bfcb1e31149b837ddb7c5 /crawler/scrape.go
parent56d7d7c020b8cb7d58c1b0543f9f1822d678f72d (diff)
downloadalkobote-4d3d10f634e872d0d4f27311c53f66680e574ad3.tar.gz
Removes Tx, because I get 'busy buffer' error. (crawler)
Diffstat (limited to 'crawler/scrape.go')
-rw-r--r--crawler/scrape.go24
1 files changed, 3 insertions, 21 deletions
diff --git a/crawler/scrape.go b/crawler/scrape.go
index 18aa4b9..d7797ca 100644
--- a/crawler/scrape.go
+++ b/crawler/scrape.go
@@ -16,9 +16,7 @@ func (app *App) ScrapeHTML(shops []Shop) {
}
- /*
- Wait until all go routines finished
- */
+ // Wait until all go routines finished
for i := 0; i < count; i++ {
<-wait
}
@@ -27,9 +25,7 @@ func (app *App) ScrapeHTML(shops []Shop) {
func (app *App) Scrape(shop Shop, wait chan bool) {
var W []Angebot
var err error
- txFailed := false
- Tx, err := app.DB.Beginx()
if err != nil {
Fatal(err, "scrape.go: Starting transaction failed. Shop: "+shop.Name)
}
@@ -45,29 +41,15 @@ func (app *App) Scrape(shop Shop, wait chan bool) {
}
}
- err = app.save_offer(Tx, W)
+ err = app.save_offer(W)
if err != nil {
- txFailed = true
Warn(err, "Saving offers failed. Shop: "+shop.Name)
}
- err = app.remove_expired(Tx, W, shop)
+ err = app.remove_expired(W, shop)
if err != nil {
- txFailed = true
Warn(err, "Removing expired offers failed. Shop: "+shop.Name)
}
- if txFailed {
- err = Tx.Rollback()
- if err != nil {
- Fatal(err, "scrape.go: Rollback transaction failed")
- }
- } else {
- err = Tx.Commit()
- if err != nil {
- Fatal(err, "scrape.go: Committing transaction failed")
- }
- }
-
wait <- true
}