diff options
| author | horus | 2023-12-18 21:21:30 +0100 |
|---|---|---|
| committer | horus | 2023-12-18 21:21:30 +0100 |
| commit | 136837ec8414591b0efccfaf74d3657aab0a08da (patch) | |
| tree | 9de9b2675a31b2f5d8dd108c3c80ad069213461c | |
| parent | de1377f669d64d2f08ae3b3432e789b98a90ead5 (diff) | |
| download | curious-crawler-136837ec8414591b0efccfaf74d3657aab0a08da.tar.gz | |
.
| -rw-r--r-- | categories.go | 2 | ||||
| -rw-r--r-- | config.go | 2 | ||||
| -rw-r--r-- | database.go | 2 | ||||
| -rw-r--r-- | init.go | 2 | ||||
| -rw-r--r-- | main.go | 7 | ||||
| -rw-r--r-- | wikipedia.go | 2 |
6 files changed, 10 insertions, 7 deletions
diff --git a/categories.go b/categories.go index 7b1fb1d..20f4b07 100644 --- a/categories.go +++ b/categories.go @@ -5,8 +5,8 @@ import ( "io/ioutil" "strings" - log "github.com/Sirupsen/logrus" "github.com/gocolly/colly" + log "github.com/sirupsen/logrus" ) func (app *App) crawlForCategories(wiki_url string) ([]string, bool) { @@ -3,7 +3,7 @@ package main import ( "os" - log "github.com/Sirupsen/logrus" + log "github.com/sirupsen/logrus" "github.com/spf13/viper" ) diff --git a/database.go b/database.go index 2852c7c..bb6997d 100644 --- a/database.go +++ b/database.go @@ -1,7 +1,7 @@ package main import ( - log "github.com/Sirupsen/logrus" + log "github.com/sirupsen/logrus" "regexp" "strconv" @@ -4,7 +4,7 @@ import ( "errors" "strings" - log "github.com/Sirupsen/logrus" + log "github.com/sirupsen/logrus" flag "github.com/spf13/pflag" ) @@ -11,9 +11,9 @@ import ( "strings" "time" - "github.com/AnikHasibul/queue" - log "github.com/Sirupsen/logrus" + "github.com/anikhasibul/queue" "github.com/jmoiron/sqlx" + log "github.com/sirupsen/logrus" ) type App struct { @@ -154,10 +154,13 @@ func (app *App) topStories() { log.Fatal(err) } + log.Debug("topStories: crawling for Categories") categories, ok := app.crawlForCategories(Story.Url) if ok { article_id := app.getArticleIdFromUrl(Story.Url) app.saveCategory(article_id, categories) + } else { + log.Warn("topStories: crawling for Categories: not ok") } } diff --git a/wikipedia.go b/wikipedia.go index 338881a..3df392d 100644 --- a/wikipedia.go +++ b/wikipedia.go @@ -8,8 +8,8 @@ import ( "io/ioutil" "net/url" - log "github.com/Sirupsen/logrus" "github.com/gocolly/colly" + log "github.com/sirupsen/logrus" ) func (app *App) crawlWikipedia(url string) { |
