diff options
Diffstat (limited to 'categories.go')
| -rw-r--r-- | categories.go | 9 |
1 files changed, 7 insertions, 2 deletions
diff --git a/categories.go b/categories.go index a92b0df..a1bd394 100644 --- a/categories.go +++ b/categories.go @@ -31,7 +31,7 @@ func (app *App) queryWMLabs(wiki_url string) ([]string, bool) { } log.Debugf("queryWMLabs: wm_url: %s", wm_url) - response := getResponse(wm_url) + response := app.getResponse(wm_url) resp_data, err := ioutil.ReadAll(response.Body) if err != nil { log.Warnf("queryWMLabs: Reading response data failed for %s", wm_url) @@ -95,7 +95,9 @@ func (app *App) crawlWMLabs(wiki_url string) (Category, bool) { } var category Category - c := colly.NewCollector() + c := colly.NewCollector( + colly.UserAgent(app.Config.UserAgent), + ) c.OnHTML(".sort-entry--wikiproject", func(e *colly.HTMLElement) { category.Name = strings.TrimSpace(e.Text) @@ -154,6 +156,9 @@ func normalizeCategory(s string) string { cat = strings.TrimSuffix(cat, "sub-project") cat = strings.TrimSuffix(cat, "Project") cat = strings.TrimSuffix(cat, "project") + if strings.ToLower(cat) == "project-independent assessment" { + return "" + } if strings.Contains(strings.ToLower(cat), "articles") { return "" |