unstructured

type Chunker interface {
	Chunk(text string) ([]string, error)
}

type ChunkingTool string

type Chunks struct {
	Text []string `json:"text"`
}

type ChunksDocument struct {
	Metadata *ChunksFileMetadata `json:"metadata"`
	Chunks   *Chunks             `json:"chunks"`
}

type ChunksFile struct {
	ConvertedDocument *ConvertedDocument `json:"convertedDocument"`
	ChunksDocument    *ChunksDocument    `json:"chunksDocument"`
}

type ChunksFileMetadata struct {
	ConvertedFileMetadata *ConvertedFileMetadata         `json:"convertedFileMetadata"`
	ChunkingTool          ChunkingTool                   `json:"chunkingTool"`
	ChunksGeneratorConfig v1alpha1.ChunksGeneratorConfig `json:"chunksGeneratorConfig"`
}

type Content struct {
	Markdown string `json:"markdown"`
}

type ConvertedDocument struct {
	Metadata *ConvertedFileMetadata `json:"metadata"`
	Content  *Content               `json:"content"`
}

type ConvertedFile struct {
	ConvertedDocument *ConvertedDocument `json:"convertedDocument"`
}

type ConvertedFileMetadata struct {
	RawFilePath       string                 `json:"rawFilePath"`
	FileIdentifier    string                 `json:"fileIdentifier"`
	DocumentConverter DocumentConverter      `json:"documentConverter"`
	DoclingConfig     v1alpha1.DoclingConfig `json:"doclingConfig"`
}

type DataSource interface {
	// SyncFilesToFilestore will store all files from the source to the filestore and return the list of file paths
	SyncFilesToFilestore(ctx context.Context, fs *filestore.FileStore) ([]RawFileMetadata, error)
}

type Destination interface {
	SyncFilesToDestination(ctx context.Context, fs *filestore.FileStore, filePaths []string) error
}

type DocumentConverter string

type EmbeddingDocument struct {
	Metadata   *EmbeddingFileMetadata `json:"metadata"`
	Embeddings []*Embeddings          `json:"embeddings"`
}

type EmbeddingFileMetadata struct {
	ConvertedFileMetadata   *ConvertedFileMetadata           `json:"convertedFileMetadata"`
	ChunkFileMetadata       *ChunksFileMetadata              `json:"chunkFileMetadata"`
	ModelName               string                           `json:"modelName"`
	NomicEmbedTextV15Config v1alpha1.NomicEmbedTextV15Config `json:"nomicEmbedTextV15Config,omitempty"`
}

type Embeddings struct {
	Text      string    `json:"text"`
	Embedding []float64 `json:"embedding"`
}

type EmbeddingsFile struct {
	ConvertedDocument *ConvertedDocument `json:"convertedDocument"`
	ChunksDocument    *ChunksDocument    `json:"chunksDocument"`
	EmbeddingDocument *EmbeddingDocument `json:"embeddingDocument"`
}

type GDriveSource struct {
	GDriveClient        *gdrive.Client
	FolderIDs           []string
	SkipFolderNames     []string
	MaxRetries          int
	ConcurrentFolders   int
	ConcurrentDownloads int
	OutputDir           string
}

type MarkdownSplitter struct {
	LangchainClient *langchain.Client
	Config          *textsplitter.Options
}

type RawFileMetadata struct {
	FilePath string `json:"filePath,omitempty"`
	UID      string `json:"uid,omitempty"`
}

type RecursiveCharacterSplitter struct {
	LangchainClient *langchain.Client
	Config          *textsplitter.Options
}

type S3BucketSource struct {
	S3Client  *s3.Client
	Bucket    string
	Prefix    string
	OutputDir string
}

type S3Destination struct {
	S3Client  *s3.Client
	Bucket    string
	Prefix    string
	StageName string
}

type TokenSplitter struct {
	LangchainClient *langchain.Client
	Config          *textsplitter.Options
}