diff options
Diffstat (limited to 'crawler/utility.go')
| -rw-r--r-- | crawler/utility.go | 6 |
1 files changed, 6 insertions, 0 deletions
diff --git a/crawler/utility.go b/crawler/utility.go index 5fa78c4..e0acf3f 100644 --- a/crawler/utility.go +++ b/crawler/utility.go @@ -10,6 +10,12 @@ import ( "github.com/gocolly/colly" ) +func customCollector(allowed_urls []string) *colly.Collector { + return colly.NewCollector( + colly.AllowedDomains(allowed_urls...), + ) +} + func stringInSlice(a string, list []string) bool { for _, b := range list { if b == a { |