generic

var (
	MS_DATE_STRING         = regexp.MustCompile(`^\d{13}$`)
	SEC_DATE_STRING        = regexp.MustCompile(`^\d{10}$`)
	CLEAN_DATE_STRING_RE   = regexp.MustCompile(`^\s*published\s*:?\s*(.*)`)
	TIME_MERIDIAN_SPACE_RE = regexp.MustCompile(`(.*\d)(am|pm)(.*)`)
	TIME_MERIDIAN_DOTS_RE  = regexp.MustCompile(`\.m\.`)
	TIME_NOW_STRING        = regexp.MustCompile(`^\s*(just|right)?\s*now\s*`)
	TIME_WITH_OFFSET_RE    = regexp.MustCompile(`-\d{3,4}$`)
)

var (
	POSITIVE_LEAD_IMAGE_URL_HINTS_RE = regexp.MustCompile("(?i)" + strings.Join(POSITIVE_LEAD_IMAGE_URL_HINTS, "|"))
	NEGATIVE_LEAD_IMAGE_URL_HINTS_RE = regexp.MustCompile("(?i)" + strings.Join(NEGATIVE_LEAD_IMAGE_URL_HINTS, "|"))
	GIF_RE                           = regexp.MustCompile(`(?i)\.gif(\?.*)?$`)
	JPG_RE                           = regexp.MustCompile(`(?i)\.jpe?g(\?.*)?$`)
	PHOTO_HINTS_RE                   = regexp.MustCompile(`(?i)figure|photo|image|caption`) // From constants.go
)

var (
	// DIGIT_RE matches any digit character
	DIGIT_RE = regexp.MustCompile(`\d`)

	// EXTRANEOUS_LINK_HINTS are words that indicate a link is probably not a next page
	EXTRANEOUS_LINK_HINTS = []string{
		"print", "archive", "comment", "discuss", "e-mail", "email",
		"share", "reply", "all", "login", "sign", "single", "adx", "entry-unrelated",
	}
	EXTRANEOUS_LINK_HINTS_RE = regexp.MustCompile(`(?i)` + strings.Join(EXTRANEOUS_LINK_HINTS, "|"))

	// NEXT_LINK_TEXT_RE matches text that likely indicates a next page link
	NEXT_LINK_TEXT_RE = regexp.MustCompile(`(?i)(next|weiter|continue|>([^|]|$)|»([^|]|$))`)

	// CAP_LINK_TEXT_RE matches text that indicates end links (first, last, etc.)
	CAP_LINK_TEXT_RE = regexp.MustCompile(`(?i)(first|last|end)`)

	// PREV_LINK_TEXT_RE matches text that indicates previous page links
	PREV_LINK_TEXT_RE = regexp.MustCompile(`(?i)(prev|earl|old|new|<|«)`)

	// PAGE_RE matches pagination-related text
	PAGE_RE = regexp.MustCompile(`(?i)pag(e|ing|inat)`)
)

var (
	// An ordered list of meta tag names that denote likely article titles.
	// All attributes should be lowercase for faster case-insensitive matching.
	// From most distinct to least distinct.
	STRONG_TITLE_META_TAGS = []string{
		"tweetmeme-title",
		"dc.title",
		"rbtitle",
		"headline",
		"title",
	}

	// og:title is weak because it typically contains context that we don't like,
	// for example the source site's name. Gotta get that brand into facebook!
	WEAK_TITLE_META_TAGS = []string{
		"og:title",
	}

	// An ordered list of CSS Selectors to find likely article titles.
	// From most explicit to least explicit.
	//
	// Note - this does not use classes like CSS. This checks to see if the string
	// exists in the className, which is not as accurate as .className (which
	// splits on spaces/endlines), but for our purposes it's close enough.
	STRONG_TITLE_SELECTORS = []string{
		".hentry .entry-title",
		"h1#articleHeader",
		"h1.articleHeader",
		"h1.article",
		".instapaper_title",
		"#meebo-title",
	}

	WEAK_TITLE_SELECTORS = []string{
		"article h1",
		"#entry-title",
		".entry-title",
		"#entryTitle",
		"#entrytitle",
		".entryTitle",
		".entrytitle",
		"#articleTitle",
		".articleTitle",
		"post post-title",
		"h1.title",
		"h2.article",
		"h1",
		"html head title",
		"title",
	}

	// Regular expression for title separators
	TITLE_SPLITTERS_RE = regexp.MustCompile(`(: | - | \| )`)

	// Domain endings regex for cleaning
	DOMAIN_ENDINGS_RE = regexp.MustCompile(`\.com$|\.net$|\.org$|\.co\.uk$`)
)

var AUTHOR_META_TAGS = []string{
	"byl",
	"clmst",
	"dc.author",
	"dcsext.author",
	"dc.creator",
	"rbauthors",
	"authors",
}

var AUTHOR_SELECTORS = []string{
	".entry .entry-author",
	".author.vcard .fn",
	".author .vcard .fn",
	".byline.vcard .fn",
	".byline .vcard .fn",
	".byline .by .author",
	".byline .by",
	".byline .author",
	".post-author.vcard",
	".post-author .vcard",
	"a[rel=author]",
	"#by_author",
	".by_author",
	"#entryAuthor",
	".entryAuthor",
	".byline a[href*=author]",
	"#author .authorname",
	".author .authorname",
	"#author",
	".author",
	".articleauthor",
	".ArticleAuthor",
	".byline",
}

var BYLINE_SELECTORS_RE = [][2]interface{}{
	{"#byline", bylineRe},
	{".byline", bylineRe},
}

var (
	// CANONICAL_META_SELECTORS - meta tag names for canonical URL extraction
	// From JavaScript: export const CANONICAL_META_SELECTORS = ['og:url'];
	CANONICAL_META_SELECTORS = []string{
		"og:url",
	}
)

var CLEAN_AUTHOR_RE = regexp.MustCompile(`(?i)^\s*(posted |written )?by\s*:?\s*(.*)`)

var DATE_PUBLISHED_META_TAGS = []string{
	"article:published_time",
	"displaydate",
	"dc.date",
	"dc.date.issued",
	"rbpubdate",
	"publish_date",
	"pub_date",
	"pagedate",
	"pubdate",
	"revision_date",
	"doc_date",
	"date_created",
	"content_create_date",
	"lastmodified",
	"created",
	"date",
}

var DATE_PUBLISHED_SELECTORS = []string{
	".hentry .dtstamp.published",
	".hentry .published",
	".hentry .dtstamp.updated",
	".hentry .updated",
	".single .published",
	".meta .published",
	".meta .postDate",
	".entry-date",
	".byline .date",
	".postmetadata .date",
	".article_datetime",
	".date-header",
	".story-date",
	".dateStamp",
	"#story .datetime",
	".dateline",
	".pubdate",
}

var DATE_PUBLISHED_URL_RES = []*regexp.Regexp{
	regexp.MustCompile(`/(20\d{2}/\d{2}/\d{2})/`),
	regexp.MustCompile(`(20\d{2}-[01]\d-[0-3]\d)`),
	regexp.MustCompile(`/(20\d{2}/(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)/[0-3]\d)/`),
}

var EXCERPT_META_SELECTORS = []string{"og:description", "twitter:description"}

var GenericDateExtractor = GenericDateExtractorType{}

var GenericTitleExtractor = struct {
	Extract func(doc *goquery.Selection, url string, metaCache []string) string
}{
	Extract: func(doc *goquery.Selection, url string, metaCache []string) string {

		html := "<html></html>"
		if doc.Length() > 0 {
			if fullHtml, err := doc.Html(); err == nil && fullHtml != "" {
				html = "<html>" + fullHtml + "</html>"
			} else {

				if doc.Parent().Length() > 0 {
					if parentHtml, err := doc.Parent().Html(); err == nil {
						html = "<html>" + parentHtml + "</html>"
					}
				}
			}
		} else {
			return ""
		}

		document, err := goquery.NewDocumentFromReader(strings.NewReader(html))
		if err != nil {
			return ""
		}

		title := dom.ExtractFromMeta(document, STRONG_TITLE_META_TAGS, metaCache, true)
		if title != nil && *title != "" {
			return cleanTitle(*title, url, doc)
		}

		title = dom.ExtractFromSelectors(doc, STRONG_TITLE_SELECTORS, 1, true)
		if title != nil && *title != "" {
			return cleanTitle(*title, url, doc)
		}

		title = dom.ExtractFromMeta(document, WEAK_TITLE_META_TAGS, metaCache, true)
		if title != nil && *title != "" {
			return cleanTitle(*title, url, doc)
		}

		title = dom.ExtractFromSelectors(doc, WEAK_TITLE_SELECTORS, 1, true)
		if title != nil && *title != "" {
			return cleanTitle(*title, url, doc)
		}

		return ""
	},
}

var GenericUrlExtractor = struct {
	Extract func(doc *goquery.Selection, url string, metaCache []string) URLResult
}{
	Extract: func(doc *goquery.Selection, url string, metaCache []string) URLResult {

		canonical := doc.Find("link[rel=canonical]")
		if canonical.Length() != 0 {
			href, exists := canonical.Attr("href")
			if exists && href != "" {
				return result(href)
			}
		}

		// Second, check for canonical URL in meta tags
		// Need to convert selection to document for meta tag extraction
		var document *goquery.Document

		if doc.Is("html") {

			if docNode := doc.Get(0); docNode != nil {
				document = goquery.NewDocumentFromNode(docNode)
			}
		}

		if document == nil {

			if html, err := doc.Html(); err == nil {
				fullHTML := html
				if !containsHTML(html) {
					fullHTML = "<html>" + html + "</html>"
				}

				if tempDoc, err := goquery.NewDocumentFromReader(strings.NewReader(fullHTML)); err == nil {
					document = tempDoc
				}
			}
		}

		if document != nil {

			metaURL := dom.ExtractFromMeta(document, CANONICAL_META_SELECTORS, metaCache, false)
			if metaURL != nil && *metaURL != "" {
				return result(*metaURL)
			}
		}

		return result(url)
	},
}

var GenericWordCountExtractor = struct {
	Extract func(options map[string]interface{}) int
}{
	Extract: func(options map[string]interface{}) int {

		if options == nil {
			return 1
		}

		contentInterface, exists := options["content"]
		if !exists {
			return 1
		}

		content, ok := contentInterface.(string)
		if !ok {
			return 1
		}

		count := getWordCount(content)

		if count == 1 {
			count = getWordCountAlt(content)
		}

		return count
	},
}

var LEAD_IMAGE_URL_META_TAGS = []string{
	"og:image",
	"twitter:image",
	"image_src",
}

var LEAD_IMAGE_URL_SELECTORS = []string{
	"link[rel=image_src]",
}

var NEGATIVE_LEAD_IMAGE_URL_HINTS = []string{
	"spacer", "sprite", "blank", "throbber", "gradient", "tile", "bg",
	"background", "icon", "social", "header", "hdr", "advert", "spinner",
	"loader", "loading", "default", "rating", "share", "facebook",
	"twitter", "theme", "promo", "ads", "wp-includes",
}

var POSITIVE_LEAD_IMAGE_URL_HINTS = []string{
	"upload",
	"wp-content",
	"large",
	"photo",
	"wp-image",
}

var SPLIT_DATE_STRING = regexp.MustCompile(`(?i)([0-9]{1,2}:[0-9]{2,2}( ?[ap].?m.?)?)|([0-9]{1,2}[/-][0-9]{1,2}[/-][0-9]{2,4})|(-[0-9]{3,4}$)|([0-9]{1,4})|(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec|january|february|march|april|may|june|july|august|september|october|november|december)`)

var TIME_AGO_STRING = regexp.MustCompile(`(\d+)\s+(seconds?|minutes?|hours?|days?|weeks?|months?|years?)\s+ago`)