nlp

type Block struct {
	Context string // parent content - e.g., sentence -> paragraph
	Line    int    // line of the block
	Scope   string // section selector
	Parent  string // parent (fully-qualfied) selector
	Text    string // text content

	// Lower is Text lower-cased, for the literal prefilter that decides
	// whether a rule's pattern can match at all. Computed once per block
	// because every rule would otherwise repeat it.
	Lower string

	// Offset is where Text begins within Context, or -1 when that is not
	// known.
	//
	// Checks that can report byte offsets need this to place a match: without
	// it, a match has to be located by searching Context for its text, which
	// finds the first occurrence rather than the one that matched. A sentence
	// repeated in a document is the common case.
	Offset int

	// Runs maps pieces of Text back to the source they were read from, for a
	// block that has no Offset of its own.
	//
	// Extraction drops inline markup, so `has <b>has</b>` arrives as `has has`
	// and the block is nowhere in Context as a whole. Its pieces are, though,
	// and each was placed as it was read. See #502.
	Runs []Run
}

type Info struct {
	Lang         string // Language of the file.
	Endpoint     string // API endpoint (optional); TODO: should this be per-file?
	Scope        string // The file's ext scope.
	Tagging      bool   // Does the file need POS tagging?
	Segmentation bool   // Does the file need sentence segmentation?
	Splitting    bool   // Does the file need paragraph splitting?
}

type IterTokenizer struct {
	// contains filtered or unexported fields
}

type Run struct {
	At, Src, N int
}

type SegmentResult struct {
	Sents []string
}

type TagResult struct {
	Tokens []tag.Token
}

type TaggedWord struct {
	Token tag.Token
	Line  int
	Span  []int
}

type TokenCache struct {
	// contains filtered or unexported fields
}