splitter

type Chunk ¶

type Chunk struct {
	// Buffer contains the actual content of the chunk
	Buffer *bytes.Buffer
	// TokenSize represents the number of tokens in this chunk
	TokenSize int
	// Start is the index of the first part in this chunk
	Start int
	// End is the index of the last part in this chunk (exclusive)
	End int
}

Chunk represents a piece of text with associated metadata for tracking its position and size within the original document.

type Graphemes ¶

type Graphemes struct {
	Options
}

func NewGraphemes(opts ...Option) *Graphemes

type GraphemesTokenCounter ¶

type GraphemesTokenCounter struct{}

func (c *GraphemesTokenCounter) Count(p []byte) int

type Markdown ¶

type Markdown struct {
	Options
	// contains filtered or unexported fields
}

Markdown markdown header text splitter.

If your origin document is HTML, you purify and convert to markdown, then split it.

func NewMarkdown(opts ...Option) *Markdown

func (s *Markdown) EnableCodeBlocks() *Markdown

func (s *Markdown) EnableHeadingHierarchy() *Markdown

func (s *Markdown) EnableReferenceLinks() *Markdown

func (s *Markdown) SetSecondSplitter(splitter embedder.Chunker) *Markdown

func (sp *Markdown) SplitText(text string) []string

type Option ¶

type Option func(*Options)

Option is a function type for configuring chunkcer Options. This follows the functional options pattern for clean and flexible configuration.

func WithBuffer(rw *bytes.Buffer) Option

func WithChunkSize(size int) Option

func WithOverlap(overlap int) Option

func WithTokenCounter(counter TokenCounter) Option

type Options ¶

type Options struct {
	// contains filtered or unexported fields
}

func (o *Options) Buffer() *bytes.Buffer

func (o *Options) Chunks() []string

func (o *Options) Read(p []byte) (int, error)

func (o *Options) Scan() error

func (o *Options) Scanner() Scanner

func (o *Options) Size() int

func (o *Options) SplitText(txt string) []string

func (o *Options) TokenCount(txt string) int

func (o *Options) Write(p []byte) (int, error)

type Phrases ¶ added in v1.2.2

type Phrases struct {
	Options
}

func NewPhrases(opts ...Option) *Phrases

type PhrasesTokenCounter ¶ added in v1.2.2

type PhrasesTokenCounter struct{}

func (c PhrasesTokenCounter) Count(p []byte) int

type Scanner ¶

type Scanner interface {
	Bytes() []byte
	Text() string
	Scan() bool
	Err() error
}

type Sentences ¶

type Sentences struct {
	Options
}

func NewSentences(opts ...Option) *Sentences

type SentencesTokenCounter ¶

type SentencesTokenCounter struct{}

func (c SentencesTokenCounter) Count(p []byte) int

type TikTokenCounter ¶

type TikTokenCounter struct {
	// contains filtered or unexported fields
}

TikTokenCounter provides accurate token counting using the tiktoken library, which implements the tokenization schemes used by OpenAI models.

func NewTikTokenCounter(encoding string) (*TikTokenCounter, error)

func (ttc *TikTokenCounter) Count(p []byte) int

type TokenCounter ¶

type TokenCounter interface {
	// Count returns the number of tokens in the given text according to the
	// implementation's tokenization strategy.
	Count(p []byte) int
}

TokenCounter defines the interface for counting tokens in a string. This abstraction allows for different tokenization strategies (e.g., words, subwords).

type Words ¶

type Words struct {
	Options
}

func NewWords(opts ...Option) *Words

type WordsTokenCounter ¶

type WordsTokenCounter struct{}

func (c WordsTokenCounter) Count(p []byte) int