path: root/crawler/utility.go

package main

import (
	"errors"
	"math"
	"math/rand"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/gocolly/colly"
)

func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

func detect_spirit_type(name string) string {
	name = strings.ToLower(name)

	matched, err := regexp.MatchString(`(^|\s)gin(\s|$)`, name)
	if err != nil {
		Fatal(err, "Gin regex failed")
	}
	if matched {
		return "Gin"
	}
	matched, err = regexp.MatchString(`(^|\s)rh?um(\s|$)`, name)
	if err != nil {
		Fatal(err, "Rum regex failed")
	}
	if matched {
		return "Rum"
	}
	matched, err = regexp.MatchString(`(^|\s)cachaca(\s|$)`, name)
	if err != nil {
		Fatal(err, "Cachaca regex failed")
	}
	if matched {
		return "Cachaca"
	}
	matched, err = regexp.MatchString(`(^|\s)([vw]odka|korn)(\s|$)`, name)
	if err != nil {
		Fatal(err, "Wodka regex failed")
	}
	if matched {
		return "Wodka"
	}
	matched, err = regexp.MatchString(`(^|\s)whiske?y(\s|$)`, name)
	if err != nil {
		Fatal(err, "Whisky regex failed")
	}
	if matched {
		return "Whisky"
	}
	matched, err = regexp.MatchString(`(^|\s)(bourbon|rye)(\s|$)`, name)
	if err != nil {
		Fatal(err, "Bourbon|Rye regex failed")
	}
	if matched {
		return "Whisky"
	}
	matched, err = regexp.MatchString(`(^|\s)weine?(\s|$)`, name)
	if err != nil {
		Fatal(err, "Wein regex failed")
	}
	if matched {
		if !strings.Contains(name, "cask") {
			return "Wein"
		}
	}
	matched, err = regexp.MatchString(`(^|\s)champagner?(\s|$)`, name)
	if err != nil {
		Fatal(err, "Champagner regex failed")
	}
	if matched {
		return "Champagner"
	}
	matched, err = regexp.MatchString(`(^|\s)(sekt|cremant|prosecco|cava)(\s|$)`, name)
	if err != nil {
		Fatal(err, "Sekt|Cremant regex failed")
	}
	if matched {
		return "Sekt"
	}
	matched, err = regexp.MatchString(`(^|\s)armagnac(\s|$)`, name)
	if err != nil {
		Fatal(err, "Armagnac regex failed")
	}
	if matched {
		return "Armagnac"
	}
	matched, err = regexp.MatchString(`(^|\s)cognac(\s|$)`, name)
	if err != nil {
		Fatal(err, "Cognac regex failed")
	}
	if matched {
		return "Cognac"
	}
	matched, err = regexp.MatchString(`(^|\s)(brandy|weinbrand)(\s|$)`, name)
	if err != nil {
		Fatal(err, "Brandy regex failed")
	}
	if matched {
		return "Brandy"
	}
	matched, err = regexp.MatchString(`(^|\s)calvados(\s|$)`, name)
	if err != nil {
		Fatal(err, "Calvados regex failed")
	}
	if matched {
		return "Calvados"
	}
	matched, err = regexp.MatchString(`(^|\s)grappa(\s|$)`, name)
	if err != nil {
		Fatal(err, "Grappa regex failed")
	}
	if matched {
		return "Grappa"
	}
	matched, err = regexp.MatchString(`(^|\s)(\wlikör|likör|likoer|liqueur)(\s|$)`, name)

	if err != nil {
		Fatal(err, "Likör regex failed")
	}
	if matched {
		return "Likör"
	}
	matched, err = regexp.MatchString(`(?i)(^|\s)(vermouth|wermut|aperitif)(\s|$|-)`, name)
	if err != nil {
		Fatal(err, "Vermouth|Wermut regex failed")
	}
	if matched {
		return "Wermut & Aperitif"
	}
	matched, err = regexp.MatchString(`(^|\s)([g|j]enever|korenwijn)(\s|$)`, name)
	if err != nil {
		Fatal(err, "Genever regex failed")
	}
	if matched {
		return "Genever"
	}
	matched, err = regexp.MatchString(`(^|\s)baijiu(\s|$)`, name)
	if err != nil {
		Fatal(err, "Baijiu regex failed")
	}
	if matched {
		return "Baijiu"
	}
	matched, err = regexp.MatchString(`(^|\s)(sherry|oloroso|fino|amontillado)(\s|$)`, name)
	if err != nil {
		Fatal(err, "Sherry regex failed")
	}
	if matched {
		if !strings.Contains(name, "cask") {
			return "Sherry"
		}
	}
	matched, err = regexp.MatchString(`((^|\s)port(wein)?(\s|$))|((^|\s)(ruby|tawny)(\s|$))`, name)
	if err != nil {
		Fatal(err, "Portwein regex failed")
	}
	if matched {
		if !strings.Contains(name, "cask") {
			return "Portwein"
		}
	}
	matched, err = regexp.MatchString(`(^|\s)(obstler|obstbrand|edelbrand|braende-geister)(\s|$)`, name) // braende-geister from spirituosen-wolf.de
	if err != nil {
		Fatal(err, "Obstbrand regex failed")
	}
	if matched {
		return "Obstbrand"
	}
	matched, err = regexp.MatchString(`(^|\s)sake(\s|$)`, name)
	if err != nil {
		Fatal(err, "Sake regex failed")
	}
	if matched {
		return "Sake"
	}
	matched, err = regexp.MatchString(`(^|\s)shochu(\s|$)`, name)
	if err != nil {
		Fatal(err, "Shochu regex failed")
	}
	if matched {
		return "Shochu"
	}
	matched, err = regexp.MatchString(`(^|\s)pisco(\s|$)`, name)
	if err != nil {
		Fatal(err, "Pisco regex failed")
	}
	if matched {
		return "Pisco"
	}
	matched, err = regexp.MatchString(`(^|\s)absint(h?)(\s|$)`, name)
	if err != nil {
		Fatal(err, "Absinth regex failed")
	}
	if matched {
		return "Absinth"
	}
	matched, err = regexp.MatchString(`(^|\s)tequila(\s|$)`, name)
	if err != nil {
		Fatal(err, "Tequila regex failed")
	}
	if matched {
		return "Tequila"
	}
	matched, err = regexp.MatchString(`(^|\s)mezcal(\s|$)`, name)
	if err != nil {
		Fatal(err, "Mezcal regex failed")
	}
	if matched {
		return "Mezcal"
	}

	return "Verschiedenes"
}

func extract_volume(volume string) (float32, error) {
	var volume_noisy string
	var is_litre_instead_of_cl bool

	// difference between cl...
	r_cl, err := regexp.Compile(`[0-9]+([,.][0-9]+)?( )?[cC][lL]`)
	if err != nil {
		Fatal(err, "Extract volume (centiliter) regex failed")
	}

	volume_noisy = r_cl.FindString(volume)

	if volume_noisy == "" {
		// ...and litre
		is_litre_instead_of_cl = true

		r_liter, err := regexp.Compile(`[0-9]+([,.][0-9]+)?( )?[lL](iter)?`)
		if err != nil {
			Fatal(err, "Extract volume regex failed")
		}
		volume_noisy = r_liter.FindString(volume)
	}

	// extract numbers
	r_liter2, err := regexp.Compile(`[0-9]+([,.][0-9]+)?`)
	if err != nil {
		Fatal(err, "2nd extract volume regex failed")
	}
	volume_noisy = r_liter2.FindString(volume_noisy)
	volume_noisy = strings.Replace(volume_noisy, ",", ".", 1)
	volume64, err := strconv.ParseFloat(volume_noisy, 0)
	if err != nil {
		Println(err, "Parsing volume to float failed")
		return 0, err
	}

	// converting from cl to litre
	if !is_litre_instead_of_cl {
		volume64 = volume64 / 100
	}

	return float32(volume64), err
}

func extract_abv(abv_noisy string) (float32, error) {
	if strings.Contains(abv_noisy, "%") {
		abv_noisy = strings.Replace(abv_noisy, "%", "", 1)
	}
	if strings.Contains(abv_noisy, "vol") {
		abv_noisy = strings.Replace(abv_noisy, "vol", "", 1)
	}
	if strings.Contains(abv_noisy, "Vol") {
		abv_noisy = strings.Replace(abv_noisy, "Vol", "", 1)
	}
	abv_noisy = strings.Replace(abv_noisy, ",", ".", 1)
	abv_noisy = strings.TrimSpace(abv_noisy)
	r_abv, err := regexp.Compile(`[0-9]+([,.][0-9]+)?`)
	if err != nil {
		Fatal(err, "Extract abv regex failed")
	}
	abv_noisy = r_abv.FindString(abv_noisy)

	abv64, err := strconv.ParseFloat(abv_noisy, 0)
	if err != nil {
		Println(err, "Parsing abv to float failed")
		return 0, err
	}

	return float32(abv64), nil
}

/*
 * In litre, but float.
 */
func get_volume(e *colly.HTMLElement) (float32, string) {

	volume_noisy := e.Request.Ctx.Get("volume")

	matched, err := regexp.MatchString(`[lL](iter)?`, volume_noisy)
	if err != nil {
		Fatal(err, "Get volume regex failed")
	}
	if !matched {
		return 0, volume_noisy
	}

	volume, err := extract_volume(volume_noisy)
	if err != nil {
		Warn(err, "Get Volume: Extract Volume failed: "+volume_noisy)
		return 0, err.Error() + " // volume_noisy: " + volume_noisy
	}

	return volume, ""
}

/*
 * In procent. (float)
 */
func get_abv(e *colly.HTMLElement) (float32, string) {

	abv_noisy := e.Request.Ctx.Get("abv")
	debug_info := e.Request.Ctx.Get("debug_info")

	if abv_noisy == "" {
		return 0, debug_info
	}
	//	abv_noisy = strings.Replace(abv_noisy, ".", ",", 1)

	abv, err := extract_abv(abv_noisy)
	if err != nil {
		Warn(err, "Get ABV: Extract ABV failed: "+abv_noisy)
		return 0, err.Error() + " // abv_noisy: " + abv_noisy
	}

	return abv, ""
}

/*
 * In cents. (int)
 */
func get_base_price(e *colly.HTMLElement) (int, error) {

	base_price_noisy := e.Request.Ctx.Get("base_price")

	if base_price_noisy == "" {
		return 0, errors.New("Base price empty")
	}

	base_price, err := sanitize_base_price(base_price_noisy)
	if err != nil {
		return 0, err
	}

	return base_price, nil
}

/*
 * Source: https://golang.org/src/math/floor.go?s=2165:2200#L104
 * Will use std lib with go version >= 1.10
 */
func RoundToEven(x float64) float64 {
	t := math.Trunc(x)
	odd := math.Remainder(t, 2) != 0
	if d := math.Abs(x - t); d > 0.5 || (d == 0.5 && odd) {
		return t + math.Copysign(1, x)
	}
	return t
}

/*
 * Returns random string with len n, used for short urls
 */
func getRandomString(n int) string {
	var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")

	rand.Seed(time.Now().UnixNano())
	s := make([]rune, n)
	for i := range s {
		s[i] = letters[rand.Intn(len(letters))]
	}
	return string(s)
}