diff options
| author | horus | 2018-06-15 23:28:18 +0200 |
|---|---|---|
| committer | horus | 2018-06-15 23:28:18 +0200 |
| commit | 8d68ac7c900241eb8499a94c23ab1f60750e7aed (patch) | |
| tree | 3a5d444f866383d5cdefc512242dc2afa236641e /crawler/utility.go | |
| parent | 0026ba55f03c5378d5773459fcdd7c6931ff42a5 (diff) | |
| download | alkobote-8d68ac7c900241eb8499a94c23ab1f60750e7aed.tar.gz | |
Introduces config for user agent, robots.txt and crawler delay. (crawler)
Diffstat (limited to 'crawler/utility.go')
| -rw-r--r-- | crawler/utility.go | 6 |
1 files changed, 0 insertions, 6 deletions
diff --git a/crawler/utility.go b/crawler/utility.go index e0acf3f..5fa78c4 100644 --- a/crawler/utility.go +++ b/crawler/utility.go @@ -10,12 +10,6 @@ import ( "github.com/gocolly/colly" ) -func customCollector(allowed_urls []string) *colly.Collector { - return colly.NewCollector( - colly.AllowedDomains(allowed_urls...), - ) -} - func stringInSlice(a string, list []string) bool { for _, b := range list { if b == a { |