path: root/main.go

package main

import (
	"encoding/json"
	"fmt"
	"html"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/anikhasibul/queue"
	"github.com/jmoiron/sqlx"
	log "github.com/sirupsen/logrus"
)

type App struct {
	Config *Config
	DB     *sqlx.DB
	Now    time.Time
}

func main() {
	var err error
	_own_conf := _conf
	app := App{Config: &_own_conf}
	_conf = Config{}

	app.Now = time.Now()

	log.Debug(fmt.Sprintf(`Connecting to "%s" database "%s" as user "%s" on host "%s:%s" with extra options "%s".`, app.Config.DBDriver, app.Config.DBDBName, app.Config.DBUser, app.Config.DBHost, app.Config.DBPort, app.Config.DBOptions))

	app.DB, err = sqlx.Connect(app.Config.DBDriver, app.Config.DBUser+":"+app.Config.DBPassword+"@tcp("+app.Config.DBHost+":"+app.Config.DBPort+")/"+app.Config.DBDBName+"?"+app.Config.DBOptions)
	if err != nil {
		log.Fatal(err, "Cannot connect to database")
	}

	if err = app.DB.Ping(); err != nil {
		log.Fatal(err, "No connection to database")
	}
	defer app.DB.Close()

	app.FixURLs()
	return
	/*
		app.deleteOrphanedArticles()
		app.topStories()
		app.deleteOrphanedArticles()
		app.updateAllDiscussions()
	*/
	app.walkDown()

	/**
	 * Resolve redirects on stored urls.
	 */
	//return
}

func (app *App) walkDown() {

	//var err error

	max_item := getMaxItem()
	//max_item := 27351341
	//max_item := 27262623
	//max_item := 41495306
	//max_item := 36128477
	//max_item := 32670334
	//max_item := 41231601
	//max_item := 41165987
	//max_item := 41136898
	//max_item := 22554000
	//max_item := 22494596
	//max_item := 22354383
	//max_item := 18984000
	//max_item := 18732000
	//max_item := 16017000
	//max_item := 15494000
	//max_item := 15038031
	//max_item := 14450000

	min_item := 0
	var new_max_item syncMaxItem

	if app.Config.OnlyUpdateStories {
		min_item = app.getMaxStoredItem()
	}
	log.Infof("walkDown: max_item: %d; min_item: %d\n", max_item, min_item)

	const maxRoutines = 400
	//const maxRoutines = 1

	q := queue.New(maxRoutines)
	defer q.Close()
	//for i := max_item; i > 22600000; i-- {
	for i := max_item; i > min_item; i-- {
		q.Add()
		go func(i int, new_max_item *syncMaxItem) {
			defer q.Done()

			Story, ok := getStory(i)
			if ok {
				if len(Story.Links) > 0 {
					//log.Debugf("%+v\n", Story)
					//log.Debugf("%+v\n", Story.Links)
				}
				err := app.saveStory(Story)
				if err != nil {
					log.Fatal(err)
				} else {
					new_max_item.mu.Lock()
					if Story.Id > new_max_item.max_item {
						new_max_item.max_item = Story.Id
					}
					new_max_item.mu.Unlock()
				}
				/*
				 */
			}

			/*
			 * Prints status update every 1000th entry
			 */
			if i%1000 == 0 {
				log.Infof("%s: Getting item %d\n", time.Now(), i)
			}
		}(i, &new_max_item)
	}
	q.Wait()

	if min_item == 0 {
		err := app.createMaxStoredItem(new_max_item.max_item)
		if err != nil {
			log.Fatal(err)
		}
	} else if min_item != new_max_item.max_item && new_max_item.max_item != 0 {
		err := app.updateNewMaxStoredItem(new_max_item.max_item)
		if err != nil {
			log.Fatal(err)
		}
	}
}

func getMaxItem() int {
	response, err := http.Get("https://hacker-news.firebaseio.com/v0/maxitem.json")
	if err != nil {
		panic(err)
	}
	data, err := ioutil.ReadAll(response.Body)
	if err != nil {
		panic(err)
	}
	max_item, err := strconv.Atoi(string(data))
	if err != nil {
		panic(err)
	}

	return max_item
}

func (app *App) topStories() {
	var err error

	data1 := strings.TrimSuffix(string(getTopStories()), "]")
	data2 := strings.TrimPrefix(string(getBestStories()), "[")

	data1 = data1 + ","
	data := data1 + data2

	var story_ids []int
	err = json.Unmarshal([]byte(data), &story_ids)
	if err != nil {
		log.Warn("topStories: Unmarshaling json failed")
		panic(err)
	}

	const maxRoutines = 20

	q := queue.New(maxRoutines)
	defer q.Close()
	for _, id := range story_ids {
		q.Add()
		go func(id int) {
			Story, ok := getStory(id)
			defer q.Done()
			if ok {
				log.Infof("%+v\n", Story)
				err = app.saveStory(Story)
				if err != nil {
					log.Fatal(err)
				}

			}
		}(id)
	}
	q.Wait()
}

func getStory(id int) (Story, bool) {
	Story := getDetail(id)
	if Story.Dead {
		return Story, false
	}
	if Story.Type == "Story" && Story.Score < 10 && Story.Descendants < 10 {
		return Story, false
	}
	var duplicates = make(map[string]bool)
	/*
		if (time.Now().Unix() - 3456000) > int64(Story.Time) {
		}
	*/

	Story.Title = stripHNPrefix(Story.Title)

	u, err := url.Parse(Story.Url)
	if err != nil {
		log.Warnf("getStory: Parsing URL failed: %s\n", err.Error())
		return Story, false
	}

	/**
	 * Check if story links to Youtube
	 */
	is_video, err := regexp.MatchString("(?i)(youtube.com)|(youtu.be)|(vimeo.com)", u.Host)
	if err != nil {
		log.Errorf("Failed to parse and match regex: %s\n", err.Error())
		return Story, false
	}
	if is_video {
		link := getURL(Story.Url, 2)
		Story.Links = append(Story.Links, link)

		log.Info("match youtube host")
		log.Infof("%+v\n", Story)

		duplicates[link.Url] = true
	}

	/**
	 * Check if story links to movie platform
	 */
	is_movie, err := regexp.MatchString("(?i)(imdb.com)|(rottentomatoes.com)|(metacritic.com)", u.Host)
	if err != nil {
		log.Errorf("Failed to parse and match regex: %s\n", err.Error())
		return Story, false
	}
	if is_movie {
		link := getURL(Story.Url, 1)
		Story.Links = append(Story.Links, link)

		log.Info("match moview platform url")
		log.Infof("%+v\n", Story)

		duplicates[link.Url] = true
	}

	/**
	 * Check for (Video) in title
	 */
	is_video, err = regexp.MatchString("(?i)(\\(video\\))|(\\[video\\])", Story.Title)
	if err != nil {
		log.Errorf("Failed to parse and match regex: %s\n", err.Error())
		return Story, false
	}
	if is_video {
		if !duplicates[Story.Url] {

			link := getURL(Story.Url, 2)
			Story.Links = append(Story.Links, link)

			log.Info("match video title")
			log.Infof("%+v\n", Story)

			duplicates[Story.Url] = true
		}

	}

	/**
	 * Check if story links to movie platform
	 */
	is_movie, err = regexp.MatchString("(?i)(imdb.com)|(rottentomatoes.com)|(metacritic.com)", u.Host)
	if err != nil {
		log.Errorf("Failed to parse and match regex: %s\n", err.Error())
		return Story, false
	}
	if is_movie {
		if !duplicates[Story.Url] {

			link := getURL(Story.Url, 1)
			Story.Links = append(Story.Links, link)

			log.Info("match moview platform url")
			log.Infof("%+v\n", Story)

			duplicates[Story.Url] = true
		}

	}

	/**
	 * Parse all URLs in Story.Text

	log.Debugf("StoryID: %d\n", Story.Id)
	log.Debugf("StoryID: %d\n", Story.Text)
	*/

	/**
	 * This comment broke my code:
	 * https://news.ycombinator.com/item?id=27351340
	 */
	tmpdoc, err := goquery.NewDocumentFromReader(strings.NewReader("<html>" + Story.Text + "</html>"))
	if err != nil {
		log.Errorf("Failed to parse html: %s\n", err.Error())
		return Story, false
	}
	sel := tmpdoc.Find("html")

	// remove all found elements from selection
	sel.Find("code").Each(func(i int, s *goquery.Selection) {
		//log.Warnf("%+v\n", s.Get(0))
		RemoveNode(sel.Get(0), s.Get(0))
	})

	tmphtml, err := sel.Html()
	if err != nil {
		log.Warn("Failed to generate html from selection: ", err.Error())
	}

	doc, err := goquery.NewDocumentFromReader(strings.NewReader(tmphtml))

	if err != nil {
		log.Errorf("Failed to parse html: %s\n", err.Error())
		return Story, false
	}

	doc.Find("a").Each(func(i int, s *goquery.Selection) {

		l, ok := s.Attr("href")

		if ok {

			/**
			 * Check for Youtube in text field
			 */
			is_video, err = regexp.MatchString("(?i)(youtube.com)|(youtu.be)|(vimeo.com)", l)
			if err != nil {
				log.Fatal("Failed to parse and match regex: %s\n", err.Error())
				//log.Errorf("Failed to parse and match regex: %s\n", err.Error())
				//return Story, false
			}
			if is_video {
				if !duplicates[l] {

					link := getURL(l, 2)
					Story.Links = append(Story.Links, link)

					log.Info("match youtube text")
					log.Infof("%+v\n", Story)

					duplicates[l] = true
				}

			}

			/**
			 * Check for movie platforms in text field
			 */
			is_movie, err = regexp.MatchString("(?i)(imdb.com)|(rottentomatoes.com)|(metacritic.com)", l)
			if err != nil {
				log.Fatal("Failed to parse and match regex: %s\n", err.Error())
				//log.Errorf("Failed to parse and match regex: %s\n", err.Error())
				//return Story, false
			}
			if is_movie {
				if !duplicates[l] {

					link := getURL(l, 1)
					Story.Links = append(Story.Links, link)

					log.Info("match moview platform text")
					log.Infof("%+v\n", Story)

					duplicates[l] = true
				}
			}
		}
	})

	//Story.Url = normalizeUrl(Story.Url)

	if len(Story.Links) > 0 {
		return Story, true
	} else {
		return Story, false
	}
}

func getURL(_url string, field int) Link {

	_url = normalizeUrl(_url)

	var link Link

	u, err := url.Parse(_url)
	if err != nil {
		log.Warnf("getURL: Parsing URL failed: %s \n", err.Error())
		return link
	}

	link.Field = field
	link.Url = _url
	link.Host = u.Host

	switch u.Host {
	case "www.youtube.com",
		"music.youtube.com",
		"www.music.youtube.com":
		//log.Warn(_url)
		if strings.HasPrefix(u.Path, "/@") || strings.HasPrefix(u.Path, "/c/") || strings.HasPrefix(u.Path, "/user/") || strings.HasPrefix(u.Path, "/channel/") {
			link.Type = "channel"

			path_parts := strings.Split(u.Path, "/")

			if strings.HasPrefix(u.Path, "/@") {
				link.Param = strings.TrimPrefix(path_parts[1], "@")
			} else {
				link.Param = path_parts[2]
			}

		} else if strings.HasPrefix(u.Path, "/shorts") || strings.HasPrefix(u.Path, "/live") || strings.HasPrefix(u.Path, "/embed") {
			link.Type = "video"

			path_parts := strings.Split(u.Path, "/")

			if len(path_parts) > 2 {
				link.Param = path_parts[2]
			} else {
				link.Param = ""
				link.Type = "null"
			}

		} else if strings.HasPrefix(u.Path, "/playlist") {
			link.Type = "playlist"
			m, err := url.ParseQuery(u.RawQuery)
			if err != nil {
				log.Warnf("getURL: Parsing RawQuery for Youtube failed: %s \n", err.Error())
				return link
			}

			/**
			 * ?list= includes the playlist id
			 */

			p, ok := m["list"]
			if !ok {
				log.Warnf("getURL: Playlist: Youtube has no param: %s \n", link.Url)
			} else {
				link.Param = p[0]
			}

		} else if strings.HasPrefix(u.Path, "/watch/") {

			link.Type = "video"
			path_parts := strings.Split(u.Path, "/")
			link.Param = path_parts[2]

		} else if strings.HasPrefix(u.Path, "/watch") {

			link.Type = "video"

			m, err := url.ParseQuery(u.RawQuery)
			if err != nil {
				log.Warnf("getURL: Parsing RawQuery for Youtube failed: %s \n", err.Error())
				return link
			}

			/**
			 * ?v= includes the video id
			 */
			p, ok := m["v"]
			if !ok {
				//log.Infof("getURL: v=VideoID: Youtube has no param: %s \n", link.Url)
				log.Warnf("getURL: v=VideoID: Youtube has no param: %s \n", link.Url)
			} else {
				link.Param = p[0]
			}
		} else if u.Path == "/" || u.Path == "" {
			link.Type = "null"
			link.Param = ""
		} else {
			link.Type = "channel"
			path_parts := strings.Split(u.Path, "/")
			link.Param = path_parts[1]
		}

		/**
		 * nice debug
		if link.Param == "" && !strings.HasPrefix(u.Path, "/clip") && !strings.HasPrefix(u.Path, "/results") && u.Path != "/watch" {
			log.Fatal(link)
		}
		*/
		break
	case "www.imdb.com":
		//log.Warn(u.Path)
		if strings.HasPrefix(u.Path, "/title/") {

			link.Type = "film"

			path_parts := strings.Split(u.Path, "/")
			if strings.HasPrefix(path_parts[1], "tt") {
				link.Param = path_parts[1]
			} else {
				log.Infof("getURL: IMDB: path_parts[1] doesn't have a film id: %s %s %+v", u.Path, path_parts[2], path_parts)
				log.Info(path_parts)
			}

		} else if strings.HasPrefix(u.Path, "/tt") {
			link.Type = "film"

			path_parts := strings.Split(u.Path, "/")
			link.Param = path_parts[1]

		} else if strings.HasPrefix(u.Path, "/name/") {

			link.Type = "actor"
			path_parts := strings.Split(u.Path, "/")
			link.Param = path_parts[2]

		} else if strings.HasPrefix(u.Path, "/character/") {

			link.Type = "character"
			path_parts := strings.Split(u.Path, "/")
			link.Param = path_parts[2]

		} else if strings.HasPrefix(u.Path, "/company/") {

			link.Type = "company"
			path_parts := strings.Split(u.Path, "/")
			link.Param = path_parts[2]

		} else if strings.HasPrefix(u.Path, "/video/") || strings.HasPrefix(u.Path, "/videoplayer/") {

			link.Type = "video"
			path_parts := strings.Split(u.Path, "/")
			link.Param = path_parts[2]

		} else if strings.HasPrefix(u.Path, "/user/") {

			link.Type = "user"
			path_parts := strings.Split(u.Path, "/")
			link.Param = path_parts[2]

		} else if strings.HasPrefix(u.Path, "/news/") || strings.HasPrefix(u.Path, "/board/announcement") {

			link.Type = "news"
			path_parts := strings.Split(u.Path, "/")
			link.Param = path_parts[2]

		} else if strings.HasPrefix(u.Path, "/review/") {

			link.Type = "review"
			path_parts := strings.Split(u.Path, "/")
			link.Param = path_parts[2]

		} else if strings.HasPrefix(u.Path, "/404") || u.Path == "" || u.Path == "/" || strings.HasPrefix(u.Path, "/interfaces") || strings.HasPrefix(u.Path, "/titanic") || strings.HasPrefix(u.Path, "/freedive/") || strings.HasPrefix(u.Path, "/conditions") || strings.HasPrefix(u.Path, "/help/") || strings.HasPrefix(u.Path, "/showtimes/") || strings.HasPrefix(u.Path, "/1") || strings.Contains(u.Path, "/mediaviewer/") || strings.HasPrefix(u.Path, "/media") || strings.HasPrefix(u.Path, "/licensing") || strings.HasPrefix(u.Path, "/lists") || strings.HasPrefix(u.Path, "/stats") || strings.HasPrefix(u.Path, "/rg/") || strings.HasPrefix(u.Path, "/hackernews") || strings.HasPrefix(u.Path, "/robots.txt") || strings.HasPrefix(u.Path, "/: the prestige") || strings.HasPrefix(u.Path, "/features/") || strings.HasPrefix(u.Path, "/keyword/") {

			link.Type = "null"
			link.Param = ""

		} else if strings.HasPrefix(u.Path, "/list/") || strings.HasPrefix(u.Path, "/chart/") || strings.Contains(u.Path, "/top-rated-") || strings.HasPrefix(u.Path, "/whats-on-tv/") {

			link.Type = "film_list"
			path_parts := strings.Split(u.Path, "/")
			link.Param = path_parts[2]

		} else if strings.HasPrefix(u.Path, "/find") || strings.HasPrefix(u.Path, "/search") || strings.HasPrefix(u.Path, "/filmosearch") {

			link.Type = "imdb_search"

		} else {
			log.Infof("getURL: IMDB: path does not include title: %s \n", u.Path)
			//log.Warnf("getURL: IMDB: path does not include title: %s \n", u.Path)
			log.Fatalf("getURL: IMDB: path does not include title: %s \n", u.Path)
		}
		break
	case "www.vimeo.com":
		// todo
	default:
		//log.Warnf("getURL: Host not recognized. Missing param: %s", u.Host)
		return link
	}

	if link.Host == "www.music.youtube.com" {
		u.Host = "music.youtube.com"
		link.Url = u.String()
		link.Host = "music.youtube.com"
	}

	return link
}

func getResponse(url string) *http.Response {
	var err error
	var response *http.Response

	response, err = http.Get(url)
	if err != nil {
		for i := 0; i < 4; i++ {
			if i == 0 {
				log.Debug("getResponse: Got error connecting to firebase/wikipedia. Retry: " + strconv.Itoa(i))
			} else {
				log.Warn("getResponse: Got error connecting to firebase/wikipedia. Retry: " + strconv.Itoa(i))
			}
			resp2, err2 := http.Get(url)
			if err2 == nil {
				return resp2
			}
		}
		panic(err)
	}
	return response
}

func getBestResponse() *http.Response {
	_url := "https://hacker-news.firebaseio.com/v0/beststories.json"
	return getResponse(_url)
}

func getTopResponse() *http.Response {
	_url := "https://hacker-news.firebaseio.com/v0/topstories.json"
	return getResponse(_url)
}

func getStoryResponse(item_id string) *http.Response {
	_url := "https://hacker-news.firebaseio.com/v0/item/" + item_id + ".json"
	return getResponse(_url)
}

func getDetail(id int) Story {
	response := getStoryResponse(strconv.Itoa(id))
	data, err := ioutil.ReadAll(response.Body)
	if err != nil {
		panic(err)
	}
	var story Story
	err = json.Unmarshal(data, &story)
	if err != nil {
		log.Warn("getDetail: Unmarshaling json failed ", data)
		panic(err)
	}
	//log.Debug("%+v\n", Story)

	story.Text = html.UnescapeString(story.Text)

	log.Tracef("StoryID: %d\n", story.Id)

	return story
}

func getTopStories() []byte {
	response := getTopResponse()
	data, err := ioutil.ReadAll(response.Body)
	if err != nil {
		panic(err)
	}

	return data
}

func getBestStories() []byte {
	response := getBestResponse()

	data, err := ioutil.ReadAll(response.Body)
	if err != nil {
		panic(err)
	}

	return data
}

func (app *App) updateAllDiscussions() {
	const maxRoutines = 20
	var item_ids []int

	app.DB.Select(&item_ids, "SELECT item_id FROM discussion WHERE posted_on > (UNIX_TIMESTAMP()-3456000) order by posted_on")

	q := queue.New(maxRoutines)
	defer q.Close()

	for _, item_id := range item_ids {
		q.Add()
		go func(item_id int) {
			defer q.Done()
			Story, ok := getStory(item_id)
			if !ok {
				/**
				 * Check if we got a network error or a dead story.
				 */
				if 0 == Story.Id {
					log.Warnf("updateAllDiscussions: Failure getting Story for item_id: %d\n", item_id)
				} else if Story.Descendants > 10 || Story.Score > 10 {
					log.Infof(`
					updateAllDiscussions: There is a bug. Can't update discussion with id %d.
					NOTE: If this is happening again, probably the url was changed from Wikipedia to a different source.
					%+v\n
					`, item_id, Story)
				}
				return
			}
			err := app.updateDiscussion(Story)
			if err != nil {
				log.Warn(err)
				return
			}
		}(item_id)
	}
	q.Wait()
}