textsplitter

package

v0.0.113 Latest Latest Go to latest Published: Aug 5, 2024 License: MIT Imports: 5 Imported by: 0

Details

Package textsplitter provides utilities for splitting and processing text.

This section is empty.

This section is empty.

This section is empty.

type BaseTextSplitter struct {
	// contains filtered or unexported fields
}

func NewBaseTextSplitter(splitTextFunc SplitTextFunc, optFns ...func(o *Options)) *BaseTextSplitter

func (ts *BaseTextSplitter) CreateDocuments(texts []string, metadatas []map[string]any) ([]schema.Document, error)

func (ts *BaseTextSplitter) SplitDocuments(docs []schema.Document) ([]schema.Document, error)

type CharacterTextSplitter struct {
	*BaseTextSplitter
	// contains filtered or unexported fields
}

func NewCharacterTextSplitter(optFns ...func(o *CharacterTextSplitterOptions)) *CharacterTextSplitter

type CharacterTextSplitterOptions struct {
	Options
	Separator string
}

type LengthFunc func(text string) int

type Options struct {
	ChunkSize     int
	ChunkOverlap  int
	KeepSeparator bool
	LengthFunc    LengthFunc
}

type RecursiveCharacterTextSplitter struct {
	*BaseTextSplitter
	// contains filtered or unexported fields
}

func NewRecusiveCharacterTextSplitter(optFns ...func(o *RecursiveCharacterTextSplitterOptions)) *RecursiveCharacterTextSplitter

type RecursiveCharacterTextSplitterOptions struct {
	Options
	Separators []string
}

type SplitTextFunc func(text string) []string