embeddings

type BertTokenizer struct {
	// contains filtered or unexported fields
}

type DownloadProgress struct {
	File            string
	BytesDownloaded int64
	TotalBytes      int64
	Percentage      float64
	Speed           float64 // MB/s
	ETA             time.Duration
}

type Downloader struct {
	// contains filtered or unexported fields
}

type Embedder struct {
	// contains filtered or unexported fields
}

type EmbedderConfig struct {
	ModelsDir    string `yaml:"models_dir"`
	DefaultModel string `yaml:"default_model"`
	BatchSize    int    `yaml:"batch_size"`
	MaxLength    int    `yaml:"max_length"`
	NumThreads   int    `yaml:"num_threads"`
	CacheEnabled bool   `yaml:"cache_enabled"`
}

type Encoding struct {
	InputIDs      []int64  `json:"input_ids"`
	AttentionMask []int64  `json:"attention_mask"`
	TokenTypeIDs  []int64  `json:"token_type_ids"`
	Tokens        []string `json:"tokens"`
}

type Manager struct {
	// contains filtered or unexported fields
}

type Model struct {
	// contains filtered or unexported fields
}

type ModelConfig struct {
	ID             string         `json:"id"`
	Organization   string         `json:"organization"`
	Name           string         `json:"name"`
	Description    string         `json:"description"`
	ModelType      string         `json:"model_type"`      // "bert", "roberta", etc.
	HiddenSize     int            `json:"hidden_size"`     // Embedding dimension
	MaxLength      int            `json:"max_length"`      // Maximum sequence length
	BaseURL        string         `json:"base_url"`        // Base URL for downloading files
	Variants       []ModelVariant `json:"variants"`        // Available model variants
	TokenizerFiles []string       `json:"tokenizer_files"` // Required tokenizer files
	ConfigFiles    []string       `json:"config_files"`    // Required config files
}

type ModelInfo struct {
	ID            string                 `json:"id"`             // e.g., "Xenova/SapBERT-from-PubMedBERT-fulltext"
	Organization  string                 `json:"organization"`   // e.g., "Xenova"
	Name          string                 `json:"name"`           // e.g., "SapBERT-from-PubMedBERT-fulltext"
	Path          string                 `json:"path"`           // Full path to model directory
	Variants      []VariantInfo          `json:"variants"`       // Available variants
	ActiveVariant string                 `json:"active_variant"` // Currently active variant
	Config        map[string]interface{} `json:"config"`         // Model configuration from config.json
	InstalledAt   time.Time              `json:"installed_at"`
	LastUsed      time.Time              `json:"last_used"`
}

type ModelVariant struct {
	Name        string `json:"name"`        // Variant name (e.g., "quantized")
	Filename    string `json:"filename"`    // Relative path to ONNX file
	Size        int64  `json:"size"`        // Expected file size in bytes
	URL         string `json:"url"`         // Download URL
	SHA256      string `json:"sha256"`      // Expected SHA256 hash
	Default     bool   `json:"default"`     // Is this the default variant?
	Description string `json:"description"` // Human-readable description
}

type ONNXEmbedder struct {
	// contains filtered or unexported fields
}

type SearchEmbedder struct {
	// contains filtered or unexported fields
}

type Tokenizer struct {
	// contains filtered or unexported fields
}

type TokenizerConfig struct {
	DoLowerCase   bool             `json:"do_lower_case"`
	MaxLength     int              `json:"model_max_length"`
	PadToken      string           `json:"pad_token"`
	SepToken      string           `json:"sep_token"`
	ClsToken      string           `json:"cls_token"`
	UnkToken      string           `json:"unk_token"`
	MaskToken     string           `json:"mask_token"`
	SpecialTokens map[string]int64 `json:"special_tokens_map"`
}

type VariantInfo struct {
	Name       string    `json:"name"`       // e.g., "quantized", "fp16", "full"
	Filename   string    `json:"filename"`   // e.g., "onnx/model_quantized.onnx"
	Size       int64     `json:"size"`       // File size in bytes
	Downloaded bool      `json:"downloaded"` // Whether this variant is downloaded
	Path       string    `json:"path"`       // Full path to the ONNX file
	CreatedAt  time.Time `json:"created_at"`
}