fix missing user agent

1 files changed, 7 insertions, 2 deletions

diff --git a/categories.go b/categories.go
index a92b0df..a1bd394 100644
--- a/categories.go
+++ b/categories.go
@@ -31,7 +31,7 @@ func (app *App) queryWMLabs(wiki_url string) ([]string, bool) {
 	}
 	log.Debugf("queryWMLabs: wm_url: %s", wm_url)
 
-	response := getResponse(wm_url)
+	response := app.getResponse(wm_url)
 	resp_data, err := ioutil.ReadAll(response.Body)
 	if err != nil {
 		log.Warnf("queryWMLabs: Reading response data failed for %s", wm_url)
@@ -95,7 +95,9 @@ func (app *App) crawlWMLabs(wiki_url string) (Category, bool) {
 	}
 
 	var category Category
-	c := colly.NewCollector()
+	c := colly.NewCollector(
+		colly.UserAgent(app.Config.UserAgent),
+	)
 
 	c.OnHTML(".sort-entry--wikiproject", func(e *colly.HTMLElement) {
 		category.Name = strings.TrimSpace(e.Text)
@@ -154,6 +156,9 @@ func normalizeCategory(s string) string {
 	cat = strings.TrimSuffix(cat, "sub-project")
 	cat = strings.TrimSuffix(cat, "Project")
 	cat = strings.TrimSuffix(cat, "project")
+	if strings.ToLower(cat) == "project-independent assessment" {
+		return ""
+	}
 
 	if strings.Contains(strings.ToLower(cat), "articles") {
 		return ""