arctl

type Dataset struct {
	Name       string `json:"name"`
	CreateTime string `json:"create_time"`

	// Parameters for embedding service
	LLMType   string `json:"llm_type"`
	LLMApiKey string `json:"llm_api_key"`

	// Parameters for vectorization
	VectorStore      string `json:"vector_store"`
	DocumentLanguage string `json:"document_language"`
	TextSplitter     string `json:"text_splitter"`
	ChunkSize        int    `json:"chunk_size"`
	ChunkOverlap     int    `json:"chunk_overlap"`

	Files map[string]File `json:"files"`
}

type File struct {
	// basic info
	Path string `json:"path"`
	Size int64  `json:"size"`

	// embedding status
	// Chunks is the number of split chunks
	Chunks int `json:"chunks"`
	// ChunksLoaded is the number of chunks loaded
	ChunksLoaded int `json:"chunks_loaded"`

	TimeCost float64 `json:"time_cost"`
}