diff options
Diffstat (limited to 'crawler/database.go')
| -rw-r--r-- | crawler/database.go | 30 |
1 files changed, 19 insertions, 11 deletions
diff --git a/crawler/database.go b/crawler/database.go index 1f38ca1..2608c59 100644 --- a/crawler/database.go +++ b/crawler/database.go @@ -2,8 +2,9 @@ package main import ( "database/sql" - "log" "strings" + + log "github.com/Sirupsen/logrus" ) func (app *App) createTables() error { @@ -59,8 +60,10 @@ func (app *App) createTables() error { view_query := `CREATE OR REPLACE VIEW ` + v + `_view AS SELECT - angebot.id, angebot.name, angebot.abv, angebot.volume, angebot.url as long_url, angebot.short_url as url,original_price/100 as original_price, discounted_price/100 as discounted_price, angebot.base_price/100 as base_price, image_url, - shop.name as shop, shop.short_url as shop_url, shop.shipping_costs/100 as shipping_costs, shop.free_shipping, ROUND(100-((discounted_price/original_price)*100)) AS procent, spirit_type, created_at + angebot.id, angebot.name, angebot.abv, angebot.volume, angebot.url as long_url, angebot.short_url as url, spirit_type, + original_price/100 as original_price, discounted_price/100 as discounted_price, angebot.base_price/100 as base_price, image_url, + shop.name as shop, shop.id as shop_id, shop.short_url as shop_url, shop.shipping_costs/100 as shipping_costs, shop.free_shipping, + ROUND(100-((discounted_price/original_price)*100)) AS procent, created_at FROM angebot JOIN shop ON angebot.shop = shop.id WHERE @@ -80,6 +83,7 @@ func (app *App) save_offer(W []Angebot) error { stmt, err := app.DB.Prepare(query) if err != nil { + Debug(err, "Save Offer: Preparing query failed") return err } defer stmt.Close() @@ -92,7 +96,7 @@ func (app *App) save_offer(W []Angebot) error { continue } - err := app.DB.QueryRow("SELECT 1 FROM angebot WHERE shop = ? AND name = ? AND url = ? AND original_price = ? AND discounted_price = ? AND spirit_type = ?", o.Shop, o.Name, o.Url, o.Original_price, o.Discounted_price, o.Spirit_type).Scan(&found) + err := app.DB.QueryRow("SELECT 1 FROM all_view WHERE shop_id = ? AND name = ? AND long_url = ? AND original_price = ? AND discounted_price = ? AND spirit_type = ?", o.Shop, o.Name, o.Url, o.Original_price, o.Discounted_price, o.Spirit_type).Scan(&found) /* */ @@ -105,6 +109,7 @@ func (app *App) save_offer(W []Angebot) error { _, err = stmt.Exec(o.Shop, o.Name, o.Url, o.Abv, o.Volume, o.Original_price, o.Discounted_price, o.Base_price, o.Valid_until, o.Image_url, o.Website, o.Spirit_type, app.Now) } if err != nil { + Debug(err, "Save Offer: Inserting offer failed") return err } @@ -115,12 +120,14 @@ func (app *App) save_offer(W []Angebot) error { return nil } -func (app *App) remove_expired(W []Angebot, shop_id int) error { +func (app *App) remove_expired(W []Angebot, shop Shop) error { - query := `SELECT id, shop, name, url, original_price, discounted_price FROM all_view WHERE shop = ? AND created_at < ?` + query := `SELECT id, shop, name, url, original_price, discounted_price FROM angebot WHERE shop = ? AND created_at < ? + AND (valid_until IS NULL OR valid_until > ?)` - rows, err := app.DB.Queryx(query, shop_id, app.Now) + rows, err := app.DB.Queryx(query, shop.Id, app.Now, app.Now) if err != nil { + Debug(err, "Remove expired: Query failed") return err } defer rows.Close() @@ -130,16 +137,20 @@ func (app *App) remove_expired(W []Angebot, shop_id int) error { err = rows.StructScan(&offer_db) if err != nil { + Debug(err, "Remove expired: Struct scan failed") return err } if !app.offer_contains(W, offer_db) { + DebugOffer(offer_db, "Contains not - Set to expire") expire_query := `UPDATE angebot SET valid_until = ? WHERE id = ?` _, err = app.DB.Exec(expire_query, app.Now, offer_db.Id) if err != nil { + Debug(err, "Remove expired: Update failed") return err } } + DebugOffer(offer_db, "Contains! DOES NOT EXPIRE!") } return nil @@ -154,8 +165,6 @@ func (app *App) offer_contains(W []Angebot, offer_db Angebot) bool { if (v.Shop == offer_db.Shop) && (v.Name == offer_db.Name) && (v.Original_price == offer_db.Original_price) && (v.Discounted_price == offer_db.Discounted_price) { if app.Config.Debug { - log.Println("Contains: " + v.Name) - log.Println("") } return true @@ -164,8 +173,7 @@ func (app *App) offer_contains(W []Angebot, offer_db Angebot) bool { } if app.Config.Debug { - log.Println("Contains not: " + offer_db.Name) - log.Println("") + log.Debug("Contains not: " + offer_db.Name) } return false |