package websearch

import (
	"io"
	"strings"

	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
)

// noise is markup that is never content: code the browser was meant to run,
// and controls pretending to be text. Dropping it is safe on any page.
var noise = map[atom.Atom]bool{
	atom.Script: true, atom.Style: true, atom.Noscript: true,
	atom.Template: true, atom.Svg: true, atom.Iframe: true,
	atom.Form: true, atom.Button: true, atom.Select: true, atom.Textarea: true,
}

// furniture is the site around the page: menus, banners, the small print. It
// is usually most of the bytes and never the reason anyone fetched the URL —
// but "usually" is why it can be put back, see HTMLToText.
var furniture = map[atom.Atom]bool{
	atom.Nav: true, atom.Header: true, atom.Footer: true, atom.Aside: true,
}

// breaking are elements that end a line. The list is short on purpose: the
// point is to keep a document's shape, not to reimplement a renderer.
var breaking = map[atom.Atom]bool{
	atom.P: true, atom.Br: true, atom.Div: true, atom.Section: true,
	atom.Article: true, atom.Li: true, atom.Tr: true, atom.Blockquote: true,
	atom.Pre: true, atom.H1: true, atom.H2: true, atom.H3: true, atom.H4: true,
	atom.H5: true, atom.H6: true, atom.Table: true, atom.Ul: true, atom.Ol: true,
	atom.Dd: true, atom.Dt: true, atom.Hr: true, atom.Figcaption: true,
}

// minUseful is how little text is so little that the reduction is more likely
// to have gone wrong than the page to have been empty.
const minUseful = 200

// HTMLToText reduces a page to the text a reader would see.
//
// It parses rather than scans, and the difference is not academic. A scanner
// has to track which elements are open by counting tags, and real pages do not
// balance: go.dev's release notes open five <nav> elements and close two. A
// skip-by-counting therefore swallows the whole document from the first
// unclosed tag onwards — which is exactly what the first version of this
// function did, reducing 69 kB of release notes to their title. html.Parse
// implements HTML5's own error recovery, the algorithm browsers use, so a
// subtree has real boundaries that an unclosed tag cannot leak past.
//
// Furniture is dropped, then put back if dropping it left almost nothing. Most
// pages keep their content out of <header> and <nav>; the few that do not
// would otherwise come back blank, and blank is the one answer a fetch tool
// must never give when there was something there.
func HTMLToText(r io.Reader) (string, error) {
	doc, err := html.Parse(r)
	if err != nil {
		return "", err
	}
	strict := tidy(render(doc, true))
	if len([]rune(strict)) >= minUseful {
		return strict, nil
	}
	if lenient := tidy(render(doc, false)); len(lenient) > len(strict) {
		return lenient, nil
	}
	return strict, nil
}

// render walks the tree, appending text and breaking the line where a block
// element begins or ends. dropFurniture is false on the second pass.
func render(root *html.Node, dropFurniture bool) string {
	var b strings.Builder
	var walk func(*html.Node)
	walk = func(n *html.Node) {
		switch n.Type {
		case html.TextNode:
			b.WriteString(n.Data)
			return
		case html.CommentNode, html.DoctypeNode:
			return
		case html.ElementNode:
			if noise[n.DataAtom] || (dropFurniture && furniture[n.DataAtom]) {
				return
			}
			// <title> lives in <head> and is not part of the page's text, but
			// it is the best one-line summary a document has, so it is kept
			// and given a line of its own.
			if breaking[n.DataAtom] || n.DataAtom == atom.Title {
				b.WriteByte('\n')
			}
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			walk(c)
		}
		if n.Type == html.ElementNode && (breaking[n.DataAtom] || n.DataAtom == atom.Title) {
			b.WriteByte('\n')
		}
	}
	walk(root)
	return b.String()
}

// tidy collapses the whitespace HTML is written with: runs of spaces inside a
// line, and every blank line between them. Without it a reduced page is mostly
// indentation.
//
// Blank lines go entirely rather than being kept one deep as a paragraph
// break. Each block element already ends its own line, so the break is not
// lost — and a list of twenty items would otherwise cost forty lines of
// someone's context window to say the same thing.
func tidy(s string) string {
	lines := strings.Split(s, "\n")
	out := make([]string, 0, len(lines))
	for _, line := range lines {
		line = strings.Join(strings.Fields(line), " ")
		if line == "" {
			continue
		}
		out = append(out, line)
	}
	return strings.TrimSpace(strings.Join(out, "\n"))
}