diff options
| author | admin | 2026-06-24 02:59:38 +0200 |
|---|---|---|
| committer | admin | 2026-06-24 02:59:38 +0200 |
| commit | d74428a39f01f4542395cea086ac241016b81a1f (patch) | |
| tree | ad27400e773205104dbd46d1358c85e0e5dc182e | |
| parent | 32369fcbd4285664a527846ed392a33bbcbc3d75 (diff) | |
| download | hncrawler-d74428a39f01f4542395cea086ac241016b81a1f.tar.gz | |
.
| -rw-r--r-- | database.go | 30 | ||||
| -rw-r--r-- | main.go | 2 |
2 files changed, 27 insertions, 5 deletions
diff --git a/database.go b/database.go index bf35f52..1252b5f 100644 --- a/database.go +++ b/database.go @@ -488,22 +488,44 @@ func (app *App) createMaxStoredItem(new_max_item int) error { } func (app *App) FixURLs() { - rows, err := app.DB.Query("SELECT url,field FROM links;") + rows, err := app.DB.Query("SELECT id, url,field FROM links where param is null;") if err != nil { log.Fatal(err) } for rows.Next() { + var link_id int var url string var field int - err = rows.Scan(&url, &field) + err = rows.Scan(&link_id, &url, &field) if err != nil { - log.Fatal(err) + log.Fatal("FixURLs: rows.Scan failed ", err) } link := getURL(url, field) - log.Printf("%+v\n", link) + //log.Printf("%+v\n", link) //log.Warnf("%+v\n", link) + + query := ` + UPDATE links + set updated_at = ?, + url = ?, + host = ?, + param = ?, + type = ? + WHERE id = ?; + ` + stmt, err := app.DB.Prepare(query) + if err != nil { + log.Fatal("FixURLs: Preparing query failed", err) + } + + _, err = stmt.Exec(app.Now, link.Url, link.Host, link.Param, link.Type, link_id) + if err != nil { + log.Fatal("FixURLs: Statement execution failed", link.Url, err) + } + + stmt.Close() } } @@ -585,7 +585,7 @@ func getURL(_url string, field int) Link { } else { log.Infof("getURL: IMDB: path does not include title: %s \n", u.Path) //log.Warnf("getURL: IMDB: path does not include title: %s \n", u.Path) - log.Fatalf("getURL: IMDB: path does not include title: %s \n", u.Path) + //log.Fatalf("getURL: IMDB: path does not include title: %s \n", u.Path) } break case "www.vimeo.com": |
