diff options
| author | admin | 2024-09-23 15:33:44 +0200 |
|---|---|---|
| committer | admin | 2024-09-23 15:33:44 +0200 |
| commit | 0b8d19eeae79030458b3e1492dd6b0c5a5bf2d75 (patch) | |
| tree | baa63f7d8c862c696affe5913553493c1f3e61c1 /database.go | |
| parent | b190512e951efdd1ed4642eed8726bf7bdf2c022 (diff) | |
| download | hncrawler-0b8d19eeae79030458b3e1492dd6b0c5a5bf2d75.tar.gz | |
saves last seen max_story_id and stops walk down there
Diffstat (limited to 'database.go')
| -rw-r--r-- | database.go | 134 |
1 files changed, 95 insertions, 39 deletions
diff --git a/database.go b/database.go index e3ba060..31594b4 100644 --- a/database.go +++ b/database.go @@ -1,10 +1,12 @@ package main import ( - log "github.com/sirupsen/logrus" "strconv" + log "github.com/sirupsen/logrus" + "database/sql" + _ "github.com/go-sql-driver/mysql" ) @@ -90,45 +92,45 @@ func (app *App) saveStory(s Story) error { return nil /* - query = ` - INSERT IGNORE discussion ( - id, - created_at, - updated_at, - article_id, - title, - source, - item_id, - source_url, - posted_on, - comments, - upvotes - ) VALUES ( - NULL, - ?, - ?, - (SELECT id FROM article WHERE url = ?), - ?, - ?, - ?, - ?, - ?, - ?, - ? - ); - ` - stmt2, err := app.DB.Prepare(query) - if err != nil { - log.Warn("saveStory: Preparing second query failed") - return err - } - defer stmt2.Close() + query = ` + INSERT IGNORE discussion ( + id, + created_at, + updated_at, + article_id, + title, + source, + item_id, + source_url, + posted_on, + comments, + upvotes + ) VALUES ( + NULL, + ?, + ?, + (SELECT id FROM article WHERE url = ?), + ?, + ?, + ?, + ?, + ?, + ?, + ? + ); + ` + stmt2, err := app.DB.Prepare(query) + if err != nil { + log.Warn("saveStory: Preparing second query failed") + return err + } + defer stmt2.Close() - _, err = stmt2.Exec(app.Now, app.Now, s.Url, s.Title, "HN", s.Id, "https://news.ycombinator.com/item?id="+strconv.Itoa(s.Id), s.Time, s.Descendants, s.Score) - if err != nil { - log.Warn("saveStory: Statement execution failed") - return err - } + _, err = stmt2.Exec(app.Now, app.Now, s.Url, s.Title, "HN", s.Id, "https://news.ycombinator.com/item?id="+strconv.Itoa(s.Id), s.Time, s.Descendants, s.Score) + if err != nil { + log.Warn("saveStory: Statement execution failed") + return err + } */ return nil @@ -424,3 +426,57 @@ func (app *App) getAllArticles() { log.Println(article_id) } } + +func (app *App) getMaxStoredItem() int { + row := app.DB.QueryRow("SELECT max_story_id FROM max_item ORDER BY id ASC LIMIT 1;") + var max_id int + err := row.Scan(&max_id) + if err == sql.ErrNoRows { + return 0 + } else if err != nil { + log.Warnf("getMaxStoredId failed\n") + log.Fatal(err) + } + return max_id +} + +func (app *App) updateNewMaxStoredItem(new_max_item int) error { + query := "UPDATE max_item SET max_story_id = ?, updated_at = ?;" + stmt, err := app.DB.Prepare(query) + if err != nil { + log.Warn("updateNewMaxStoredItem: Preparing query failed") + return err + } + defer stmt.Close() + + _, err = stmt.Exec(new_max_item, app.Now) + if err != nil { + log.Warnf("updateNewMaxStoredItem: Statement execution failed: %d\n", new_max_item) + return err + } + log.Infof("updateNewMaxStoredItem: updated max_story_id to: %d\n", new_max_item) + + return nil +} + +/** + * Creates new max_story_id + */ +func (app *App) createMaxStoredItem(new_max_item int) error { + stmt, err := app.DB.Prepare("INSERT INTO max_item VALUES (null, ?, ?, ?);") + if err != nil { + log.Warn("getMaxStoredItem: Preparing query failed") + return err + } + defer stmt.Close() + + _, err = stmt.Exec(new_max_item, app.Now, app.Now) + if err != nil { + log.Warnf("getMaxStoredItem: Executing query failed with new_max_item: %d\n", new_max_item) + return err + } + + log.Info("getMaxStoredItem: creatd MaxStoredItem for the first time") + + return nil +} |
