diff options
| -rw-r--r-- | crawler/database.go | 18 | ||||
| -rw-r--r-- | crawler/sanitize.go | 2 | ||||
| -rw-r--r-- | crawler/scrape.go | 24 |
3 files changed, 12 insertions, 32 deletions
diff --git a/crawler/database.go b/crawler/database.go index e47f723..6e08b78 100644 --- a/crawler/database.go +++ b/crawler/database.go @@ -4,8 +4,6 @@ import ( "database/sql" "fmt" "strings" - - "github.com/jmoiron/sqlx" ) func (app *App) createTables() error { @@ -91,13 +89,13 @@ func (app *App) createTables() error { } /** - * Saves scrapped offers in database. Detects which offers are new. Runs inside tx. + * Saves scrapped offers in database. Detects which offers are new. */ -func (app *App) save_offer(Tx *sqlx.Tx, W []Angebot) error { +func (app *App) save_offer(W []Angebot) error { query := `INSERT INTO angebot (shop, name, url, abv, volume, age, original_price, discounted_price, base_price, valid_until, image_url, website_raw, spirit_type, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - stmt, err := Tx.Prepare(query) + stmt, err := app.DB.Prepare(query) if err != nil { Warn(err, "Save Offer: Preparing query failed") return err @@ -114,7 +112,7 @@ func (app *App) save_offer(Tx *sqlx.Tx, W []Angebot) error { // resembles UNIQUE constraint detect_duplicate_query := fmt.Sprintf(`SELECT 1 FROM _intern_view WHERE name = ? AND shop_id = %d AND volume = %4.2f AND abv = %4.2f AND original_price = %d AND discounted_price = %d AND valid_until = %d`, o.Shop, o.Volume, o.Abv, o.Original_price, o.Discounted_price, o.Valid_until) - err := Tx.QueryRow(detect_duplicate_query, o.Name).Scan(&found) + err := app.DB.QueryRow(detect_duplicate_query, o.Name).Scan(&found) if err == sql.ErrNoRows { @@ -144,14 +142,14 @@ func (app *App) save_offer(Tx *sqlx.Tx, W []Angebot) error { } /** - * Invalidates expired offers by updating 'valid_until' column. Detects which offers are expired. Runs inside Tx. + * Invalidates expired offers by updating 'valid_until' column. Detects which offers are expired. */ -func (app *App) remove_expired(Tx *sqlx.Tx, W []Angebot, shop Shop) error { +func (app *App) remove_expired(W []Angebot, shop Shop) error { query := `SELECT id, name, shop, volume, abv, original_price, discounted_price FROM angebot WHERE shop = ? AND created_at < ? AND (valid_until = 0 OR valid_until > ?)` - rows, err := Tx.Queryx(query, shop.Id, app.Now, app.Now) + rows, err := app.DB.Queryx(query, shop.Id, app.Now, app.Now) if err != nil { Warn(err, "Remove expired: Query failed") return err @@ -170,7 +168,7 @@ func (app *App) remove_expired(Tx *sqlx.Tx, W []Angebot, shop Shop) error { if !app.offer_contains(W, offer_db) { DebugOffer(offer_db, "Contains not - Set to expire") expire_query := `UPDATE angebot SET valid_until = ? WHERE id = ?` - _, err = Tx.Exec(expire_query, app.Now, offer_db.Id) + _, err = app.DB.Exec(expire_query, app.Now, offer_db.Id) if err != nil { offer_db.error_msg = err.Error() offer_db.error_ctx = fmt.Sprintf("UPDATE angebot SET valid_until = %d WHERE id = %d", app.Now, offer_db.Id) diff --git a/crawler/sanitize.go b/crawler/sanitize.go index 4969be9..7bfbc03 100644 --- a/crawler/sanitize.go +++ b/crawler/sanitize.go @@ -49,7 +49,7 @@ func sanitize_offer(angebote []Angebot, shop Shop, try int) []Angebot { continue } - //offer.Website = "" + offer.Website = "" W = append(W, offer) } diff --git a/crawler/scrape.go b/crawler/scrape.go index 18aa4b9..d7797ca 100644 --- a/crawler/scrape.go +++ b/crawler/scrape.go @@ -16,9 +16,7 @@ func (app *App) ScrapeHTML(shops []Shop) { } - /* - Wait until all go routines finished - */ + // Wait until all go routines finished for i := 0; i < count; i++ { <-wait } @@ -27,9 +25,7 @@ func (app *App) ScrapeHTML(shops []Shop) { func (app *App) Scrape(shop Shop, wait chan bool) { var W []Angebot var err error - txFailed := false - Tx, err := app.DB.Beginx() if err != nil { Fatal(err, "scrape.go: Starting transaction failed. Shop: "+shop.Name) } @@ -45,29 +41,15 @@ func (app *App) Scrape(shop Shop, wait chan bool) { } } - err = app.save_offer(Tx, W) + err = app.save_offer(W) if err != nil { - txFailed = true Warn(err, "Saving offers failed. Shop: "+shop.Name) } - err = app.remove_expired(Tx, W, shop) + err = app.remove_expired(W, shop) if err != nil { - txFailed = true Warn(err, "Removing expired offers failed. Shop: "+shop.Name) } - if txFailed { - err = Tx.Rollback() - if err != nil { - Fatal(err, "scrape.go: Rollback transaction failed") - } - } else { - err = Tx.Commit() - if err != nil { - Fatal(err, "scrape.go: Committing transaction failed") - } - } - wait <- true } |