summaryrefslogtreecommitdiff
path: root/main.go
diff options
context:
space:
mode:
authorhorus2023-12-18 21:21:30 +0100
committerhorus2023-12-18 21:21:30 +0100
commit136837ec8414591b0efccfaf74d3657aab0a08da (patch)
tree9de9b2675a31b2f5d8dd108c3c80ad069213461c /main.go
parentde1377f669d64d2f08ae3b3432e789b98a90ead5 (diff)
downloadcurious-crawler-136837ec8414591b0efccfaf74d3657aab0a08da.tar.gz
.
Diffstat (limited to 'main.go')
-rw-r--r--main.go7
1 files changed, 5 insertions, 2 deletions
diff --git a/main.go b/main.go
index 84811fe..ba9b6c4 100644
--- a/main.go
+++ b/main.go
@@ -11,9 +11,9 @@ import (
"strings"
"time"
- "github.com/AnikHasibul/queue"
- log "github.com/Sirupsen/logrus"
+ "github.com/anikhasibul/queue"
"github.com/jmoiron/sqlx"
+ log "github.com/sirupsen/logrus"
)
type App struct {
@@ -154,10 +154,13 @@ func (app *App) topStories() {
log.Fatal(err)
}
+ log.Debug("topStories: crawling for Categories")
categories, ok := app.crawlForCategories(Story.Url)
if ok {
article_id := app.getArticleIdFromUrl(Story.Url)
app.saveCategory(article_id, categories)
+ } else {
+ log.Warn("topStories: crawling for Categories: not ok")
}
}