summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorhorus2023-12-18 21:21:30 +0100
committerhorus2023-12-18 21:21:30 +0100
commit136837ec8414591b0efccfaf74d3657aab0a08da (patch)
tree9de9b2675a31b2f5d8dd108c3c80ad069213461c
parentde1377f669d64d2f08ae3b3432e789b98a90ead5 (diff)
downloadcurious-crawler-136837ec8414591b0efccfaf74d3657aab0a08da.tar.gz
.
-rw-r--r--categories.go2
-rw-r--r--config.go2
-rw-r--r--database.go2
-rw-r--r--init.go2
-rw-r--r--main.go7
-rw-r--r--wikipedia.go2
6 files changed, 10 insertions, 7 deletions
diff --git a/categories.go b/categories.go
index 7b1fb1d..20f4b07 100644
--- a/categories.go
+++ b/categories.go
@@ -5,8 +5,8 @@ import (
"io/ioutil"
"strings"
- log "github.com/Sirupsen/logrus"
"github.com/gocolly/colly"
+ log "github.com/sirupsen/logrus"
)
func (app *App) crawlForCategories(wiki_url string) ([]string, bool) {
diff --git a/config.go b/config.go
index 14c75ee..485654a 100644
--- a/config.go
+++ b/config.go
@@ -3,7 +3,7 @@ package main
import (
"os"
- log "github.com/Sirupsen/logrus"
+ log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
diff --git a/database.go b/database.go
index 2852c7c..bb6997d 100644
--- a/database.go
+++ b/database.go
@@ -1,7 +1,7 @@
package main
import (
- log "github.com/Sirupsen/logrus"
+ log "github.com/sirupsen/logrus"
"regexp"
"strconv"
diff --git a/init.go b/init.go
index 7ce40c5..9127c5d 100644
--- a/init.go
+++ b/init.go
@@ -4,7 +4,7 @@ import (
"errors"
"strings"
- log "github.com/Sirupsen/logrus"
+ log "github.com/sirupsen/logrus"
flag "github.com/spf13/pflag"
)
diff --git a/main.go b/main.go
index 84811fe..ba9b6c4 100644
--- a/main.go
+++ b/main.go
@@ -11,9 +11,9 @@ import (
"strings"
"time"
- "github.com/AnikHasibul/queue"
- log "github.com/Sirupsen/logrus"
+ "github.com/anikhasibul/queue"
"github.com/jmoiron/sqlx"
+ log "github.com/sirupsen/logrus"
)
type App struct {
@@ -154,10 +154,13 @@ func (app *App) topStories() {
log.Fatal(err)
}
+ log.Debug("topStories: crawling for Categories")
categories, ok := app.crawlForCategories(Story.Url)
if ok {
article_id := app.getArticleIdFromUrl(Story.Url)
app.saveCategory(article_id, categories)
+ } else {
+ log.Warn("topStories: crawling for Categories: not ok")
}
}
diff --git a/wikipedia.go b/wikipedia.go
index 338881a..3df392d 100644
--- a/wikipedia.go
+++ b/wikipedia.go
@@ -8,8 +8,8 @@ import (
"io/ioutil"
"net/url"
- log "github.com/Sirupsen/logrus"
"github.com/gocolly/colly"
+ log "github.com/sirupsen/logrus"
)
func (app *App) crawlWikipedia(url string) {