From 13b1e31cb91f5af98951f4353d49429f356d7f12 Mon Sep 17 00:00:00 2001
From: horus
Date: Mon, 19 Feb 2018 22:53:26 +0100
Subject: Saving scraped offers runs in Tx. (crawler)

---
 crawler/scrape.go | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

(limited to 'crawler/scrape.go')

diff --git a/crawler/scrape.go b/crawler/scrape.go
index 055b00e..6ef9fcf 100644
--- a/crawler/scrape.go
+++ b/crawler/scrape.go
@@ -27,6 +27,12 @@ func (app *App) ScrapeHTML(shops []Shop) {
 func (app *App) Scrape(shop Shop, wait chan bool) {
 	var W []Angebot
 	var err error
+	txFailed := false
+
+	app.Tx, err = app.DB.Beginx()
+	if err != nil {
+		Fatal(err, "scrape.go: Starting transaction failed. Shop: "+shop.Name)
+	}
 
 	// retry on error
 	for i := 0; i < 3; i++ {
@@ -41,11 +47,25 @@ func (app *App) Scrape(shop Shop, wait chan bool) {
 
 	err = app.save_offer(W)
 	if err != nil {
-		Warn(err, "Saving offers failed")
+		txFailed = true
+		Warn(err, "Saving offers failed. Shop: "+shop.Name)
 	}
 	err = app.remove_expired(W, shop)
 	if err != nil {
-		Warn(err, "Removing expired offers failed")
+		txFailed = true
+		Warn(err, "Removing expired offers failed. Shop: "+shop.Name)
+	}
+
+	if txFailed {
+		err = app.Tx.Rollback()
+		if err != nil {
+			Fatal(err, "scrape.go: Rollback transaction failed")
+		}
+	} else {
+		err = app.Tx.Commit()
+		if err != nil {
+			Fatal(err, "scrape.go: Committing transaction failed")
+		}
 	}
 
 	wait <- true
-- 
cgit v1.2.3