tokenizerpb

type ChatMessage struct {
	Role          string         `protobuf:"bytes,1,opt,name=role,proto3" json:"role,omitempty"`                                                // Role of the message (e.g., "user", "assistant", "system")
	Content       *string        `protobuf:"bytes,2,opt,name=content,proto3,oneof" json:"content,omitempty"`                                    // Plain-text content (Content.Raw)
	ContentParts  []*ContentPart `protobuf:"bytes,3,rep,name=content_parts,json=contentParts,proto3" json:"content_parts,omitempty"`            // Multimodal content blocks (Content.Structured)
	ToolCallsJson *string        `protobuf:"bytes,4,opt,name=tool_calls_json,json=toolCallsJson,proto3,oneof" json:"tool_calls_json,omitempty"` // Assistant tool calls, encoded as a JSON array
	// contains filtered or unexported fields
}

type ChatTemplateRequest struct {
	ConversationTurns         []*ConversationTurn `protobuf:"bytes,1,rep,name=conversation_turns,json=conversationTurns,proto3" json:"conversation_turns,omitempty"` // The conversation turns (batches of messages)
	Tools                     []*ToolDescription  `protobuf:"bytes,2,rep,name=tools,proto3" json:"tools,omitempty"`                                                  // Tools available to the conversation
	Documents                 []*Document         `protobuf:"bytes,3,rep,name=documents,proto3" json:"documents,omitempty"`                                          // Documents related to the conversation
	ChatTemplate              string              `protobuf:"bytes,4,opt,name=chat_template,json=chatTemplate,proto3" json:"chat_template,omitempty"`                // The chat template to use
	ReturnAssistantTokensMask bool                ``                                                                                                                 // Whether to return assistant token mask
	/* 141-byte string literal not displayed */
	ContinueFinalMessage bool              `protobuf:"varint,6,opt,name=continue_final_message,json=continueFinalMessage,proto3" json:"continue_final_message,omitempty"` // Whether to continue the final message
	AddGenerationPrompt  bool              `protobuf:"varint,7,opt,name=add_generation_prompt,json=addGenerationPrompt,proto3" json:"add_generation_prompt,omitempty"`    // Whether to add generation prompt
	ChatTemplateKwargs   map[string]*Value ``                                                                                                                             // Additional chat template arguments
	/* 191-byte string literal not displayed */
	ModelName string `protobuf:"bytes,9,opt,name=model_name,json=modelName,proto3" json:"model_name,omitempty"` // The name of the model to use for tokenization
	// contains filtered or unexported fields
}

type ChatTemplateResponse struct {
	RenderedPrompt string `protobuf:"bytes,1,opt,name=rendered_prompt,json=renderedPrompt,proto3" json:"rendered_prompt,omitempty"` // The rendered chat template prompt
	Success        bool   `protobuf:"varint,2,opt,name=success,proto3" json:"success,omitempty"`                                    // Whether the request was successful
	ErrorMessage   string `protobuf:"bytes,3,opt,name=error_message,json=errorMessage,proto3" json:"error_message,omitempty"`       // Error message if the request failed
	// contains filtered or unexported fields
}

type ContentPart struct {
	Type     string    `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"`                               // "text" or "image_url"
	Text     *string   `protobuf:"bytes,2,opt,name=text,proto3,oneof" json:"text,omitempty"`                         // Text content (type == "text")
	ImageUrl *ImageUrl `protobuf:"bytes,3,opt,name=image_url,json=imageUrl,proto3,oneof" json:"image_url,omitempty"` // Image content (type == "image_url")
	// contains filtered or unexported fields
}

type ConversationTurn struct {
	Messages []*ChatMessage `protobuf:"bytes,1,rep,name=messages,proto3" json:"messages,omitempty"` // The messages in this turn
	// contains filtered or unexported fields
}

type Document struct {
	Document map[string]*Value `` // Document definition
	/* 143-byte string literal not displayed */
	// contains filtered or unexported fields
}

type ImageUrl struct {
	Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` // URL of the image (http/https or data URI)
	// contains filtered or unexported fields
}

type InitializeTokenizerRequest struct {
	ModelName           string `protobuf:"bytes,1,opt,name=model_name,json=modelName,proto3" json:"model_name,omitempty"`                                  // The name of the model to initialize
	EnableThinking      bool   `protobuf:"varint,2,opt,name=enable_thinking,json=enableThinking,proto3" json:"enable_thinking,omitempty"`                  // Whether to enable thinking tokens
	AddGenerationPrompt bool   `protobuf:"varint,3,opt,name=add_generation_prompt,json=addGenerationPrompt,proto3" json:"add_generation_prompt,omitempty"` // Whether to add generation prompt
	// contains filtered or unexported fields
}

type InitializeTokenizerResponse struct {
	Success      bool   `protobuf:"varint,1,opt,name=success,proto3" json:"success,omitempty"`                              // Whether the initialization was successful
	ErrorMessage string `protobuf:"bytes,2,opt,name=error_message,json=errorMessage,proto3" json:"error_message,omitempty"` // Error message if initialization failed
	// contains filtered or unexported fields
}

type ListValue struct {
	Values []*Value `protobuf:"bytes,1,rep,name=values,proto3" json:"values,omitempty"`
	// contains filtered or unexported fields
}

type MultiModalFeatures struct {
	MmHashes map[string]*StringList `` // Per-modality item hashes
	/* 159-byte string literal not displayed */
	MmPlaceholders map[string]*PlaceholderRangeList `` // Per-modality placeholder ranges
	/* 177-byte string literal not displayed */
	// contains filtered or unexported fields
}

type PlaceholderRange struct {
	Offset int32 `protobuf:"varint,1,opt,name=offset,proto3" json:"offset,omitempty"` // Start index of the placeholder tokens in the prompt
	Length int32 `protobuf:"varint,2,opt,name=length,proto3" json:"length,omitempty"` // Number of placeholder tokens
	// contains filtered or unexported fields
}

type PlaceholderRangeList struct {
	Ranges []*PlaceholderRange `protobuf:"bytes,1,rep,name=ranges,proto3" json:"ranges,omitempty"`
	// contains filtered or unexported fields
}

type RenderChatCompletionRequest struct {
	ModelName           string         `protobuf:"bytes,1,opt,name=model_name,json=modelName,proto3" json:"model_name,omitempty"`          // Model name to use for renderer selection
	Messages            []*ChatMessage `protobuf:"bytes,2,rep,name=messages,proto3" json:"messages,omitempty"`                             // Conversation messages
	ToolsJson           *string        `protobuf:"bytes,3,opt,name=tools_json,json=toolsJson,proto3,oneof" json:"tools_json,omitempty"`    // Tools available to the model, as a JSON array
	ChatTemplate        string         `protobuf:"bytes,4,opt,name=chat_template,json=chatTemplate,proto3" json:"chat_template,omitempty"` // Override chat template (optional)
	AddGenerationPrompt *bool          ``                                                                                                  // Whether to append the generation prompt (default: true)
	/* 127-byte string literal not displayed */
	ContinueFinalMessage bool    `protobuf:"varint,6,opt,name=continue_final_message,json=continueFinalMessage,proto3" json:"continue_final_message,omitempty"` // Whether to continue the final message
	ChatTemplateKwargs   *string `protobuf:"bytes,7,opt,name=chat_template_kwargs,json=chatTemplateKwargs,proto3,oneof" json:"chat_template_kwargs,omitempty"`  // Additional chat template arguments as JSON
	// contains filtered or unexported fields
}

type RenderChatCompletionResponse struct {
	RequestId    string              `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`          // Request ID from the render response
	TokenIds     []uint32            `protobuf:"varint,2,rep,packed,name=token_ids,json=tokenIds,proto3" json:"token_ids,omitempty"`     // Token IDs for the rendered input
	Features     *MultiModalFeatures `protobuf:"bytes,3,opt,name=features,proto3" json:"features,omitempty"`                             // Multimodal features (populated for MM inputs)
	Success      bool                `protobuf:"varint,4,opt,name=success,proto3" json:"success,omitempty"`                              // Whether the request was successful
	ErrorMessage string              `protobuf:"bytes,5,opt,name=error_message,json=errorMessage,proto3" json:"error_message,omitempty"` // Error message if the request failed
	// contains filtered or unexported fields
}

type RenderCompletionRequest struct {
	ModelName string `protobuf:"bytes,1,opt,name=model_name,json=modelName,proto3" json:"model_name,omitempty"` // Model name to use for renderer selection
	Prompt    string `protobuf:"bytes,2,opt,name=prompt,proto3" json:"prompt,omitempty"`                        // Text prompt to render
	// contains filtered or unexported fields
}

type RenderCompletionResponse struct {
	RequestId    string   `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`          // Request ID from the render response
	TokenIds     []uint32 `protobuf:"varint,2,rep,packed,name=token_ids,json=tokenIds,proto3" json:"token_ids,omitempty"`     // Token IDs for the rendered prompt
	Success      bool     `protobuf:"varint,3,opt,name=success,proto3" json:"success,omitempty"`                              // Whether the request was successful
	ErrorMessage string   `protobuf:"bytes,4,opt,name=error_message,json=errorMessage,proto3" json:"error_message,omitempty"` // Error message if the request failed
	// contains filtered or unexported fields
}

type StringList struct {
	Values []string `protobuf:"bytes,1,rep,name=values,proto3" json:"values,omitempty"`
	// contains filtered or unexported fields
}

type StructValue struct {
	Fields map[string]*Value `` /* 139-byte string literal not displayed */
	// contains filtered or unexported fields
}

type TokenizationServiceClient interface {
	// Tokenize converts a text input to token IDs
	Tokenize(ctx context.Context, in *TokenizeRequest, opts ...grpc.CallOption) (*TokenizeResponse, error)
	// Deprecated: use RenderChatCompletion instead.
	RenderChatTemplate(ctx context.Context, in *ChatTemplateRequest, opts ...grpc.CallOption) (*ChatTemplateResponse, error)
	// InitializeTokenizer initializes the tokenizer for a specific model
	InitializeTokenizer(ctx context.Context, in *InitializeTokenizerRequest, opts ...grpc.CallOption) (*InitializeTokenizerResponse, error)
	// RenderChatCompletion renders an OpenAI chat completion request via OpenAIServingRender,
	// returning token IDs and multimodal features as a GenerateRequest-compatible response
	RenderChatCompletion(ctx context.Context, in *RenderChatCompletionRequest, opts ...grpc.CallOption) (*RenderChatCompletionResponse, error)
	// RenderCompletion renders an OpenAI completion request via OpenAIServingRender,
	// returning the rendered output for the given prompt
	RenderCompletion(ctx context.Context, in *RenderCompletionRequest, opts ...grpc.CallOption) (*RenderCompletionResponse, error)
}

type TokenizationServiceServer interface {
	// Tokenize converts a text input to token IDs
	Tokenize(context.Context, *TokenizeRequest) (*TokenizeResponse, error)
	// Deprecated: use RenderChatCompletion instead.
	RenderChatTemplate(context.Context, *ChatTemplateRequest) (*ChatTemplateResponse, error)
	// InitializeTokenizer initializes the tokenizer for a specific model
	InitializeTokenizer(context.Context, *InitializeTokenizerRequest) (*InitializeTokenizerResponse, error)
	// RenderChatCompletion renders an OpenAI chat completion request via OpenAIServingRender,
	// returning token IDs and multimodal features as a GenerateRequest-compatible response
	RenderChatCompletion(context.Context, *RenderChatCompletionRequest) (*RenderChatCompletionResponse, error)
	// RenderCompletion renders an OpenAI completion request via OpenAIServingRender,
	// returning the rendered output for the given prompt
	RenderCompletion(context.Context, *RenderCompletionRequest) (*RenderCompletionResponse, error)
	// contains filtered or unexported methods
}

type TokenizeRequest struct {
	Input            string `protobuf:"bytes,1,opt,name=input,proto3" json:"input,omitempty"`                                                  // The text input to tokenize
	ModelName        string `protobuf:"bytes,2,opt,name=model_name,json=modelName,proto3" json:"model_name,omitempty"`                         // The name of the model to use for tokenization
	AddSpecialTokens bool   `protobuf:"varint,3,opt,name=add_special_tokens,json=addSpecialTokens,proto3" json:"add_special_tokens,omitempty"` // Whether to add special tokens during tokenization
	// contains filtered or unexported fields
}

type TokenizeResponse struct {
	InputIds     []uint32 `protobuf:"varint,1,rep,packed,name=input_ids,json=inputIds,proto3" json:"input_ids,omitempty"`     // Token IDs for the input
	Success      bool     `protobuf:"varint,2,opt,name=success,proto3" json:"success,omitempty"`                              // Whether the request was successful
	ErrorMessage string   `protobuf:"bytes,3,opt,name=error_message,json=errorMessage,proto3" json:"error_message,omitempty"` // Error message if the request failed
	// Direct array of [start, end] pairs
	OffsetPairs []uint32 `protobuf:"varint,4,rep,packed,name=offset_pairs,json=offsetPairs,proto3" json:"offset_pairs,omitempty"` // Flattened array of [start, end, start, end, ...]
	// contains filtered or unexported fields
}

type ToolDescription struct {
	Tool map[string]*Value `` // Tool definition
	/* 135-byte string literal not displayed */
	// contains filtered or unexported fields
}

type UnimplementedTokenizationServiceServer struct{}

type UnsafeTokenizationServiceServer interface {
	// contains filtered or unexported methods
}

type Value struct {

	// Types that are valid to be assigned to Value:
	//
	//	*Value_StringValue
	//	*Value_NumberValue
	//	*Value_BoolValue
	//	*Value_ListValue
	//	*Value_StructValue
	Value isValue_Value `protobuf_oneof:"value"`
	// contains filtered or unexported fields
}

type Value_BoolValue struct {
	BoolValue bool `protobuf:"varint,3,opt,name=bool_value,json=boolValue,proto3,oneof"`
}

type Value_ListValue struct {
	ListValue *ListValue `protobuf:"bytes,4,opt,name=list_value,json=listValue,proto3,oneof"`
}

type Value_NumberValue struct {
	NumberValue float64 `protobuf:"fixed64,2,opt,name=number_value,json=numberValue,proto3,oneof"`
}

type Value_StringValue struct {
	StringValue string `protobuf:"bytes,1,opt,name=string_value,json=stringValue,proto3,oneof"`
}

type Value_StructValue struct {
	StructValue *StructValue `protobuf:"bytes,5,opt,name=struct_value,json=structValue,proto3,oneof"`
}