agent

package
v0.1.6 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 8, 2026 License: MIT Imports: 50 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	UserStateSpeaking  UserState = "speaking"
	UserStateListening UserState = "listening"
	UserStateAway      UserState = "away"

	AgentStateInitializing AgentState = "initializing"
	AgentStateIdle         AgentState = "idle"
	AgentStateListening    AgentState = "listening"
	AgentStateThinking     AgentState = "thinking"
	AgentStateSpeaking     AgentState = "speaking"
)
View Source 
const (
	SpeechPriorityLow    = 0
	SpeechPriorityNormal = 5
	SpeechPriorityHigh   = 10
	InterruptionTimeout  = 5 * time.Second
)

Variables

View Source 
var (
	ErrAgentSessionNotRunning       = errors.New("AgentSession isn't running")
	ErrAgentSessionNestedRun        = errors.New("nested runs are not supported")
	ErrAgentSessionUserdataNotSet   = errors.New("AgentSession userdata is not set")
	ErrAgentSessionJobContextNotSet = errors.New("agent session job context is not set")
)
View Source 
var (
	ErrRunResultNotDone         = errors.New("cannot retrieve final_output, RunResult is not done")
	ErrRunResultNoFinalOutput   = errors.New("no final output")
	ErrRunResultFinalOutputType = errors.New("run result final output type mismatch")
)
View Source 
var (
	ErrSpeechInterruptionsDisabled = speechHandleReferenceError("This generation handle does not allow interruptions")
	ErrSpeechAlreadyInterrupted    = speechHandleReferenceError("Cannot set allow_interruptions to False, the SpeechHandle is already interrupted")
	ErrSpeechInterrupted           = errors.New("speech interrupted")
	ErrSpeechNoActiveGeneration    = errors.New("speech handle has no active generation")
)
View Source 
var ErrAgentTaskAlreadyDone = errors.New("AgentTask is already done")
View Source 
var ErrAgentTaskAlreadyWaited = errors.New("AgentTask is not re-entrant, await only once")
View Source 
var ErrFunctionToolEventLengthMismatch = functionToolEventLengthMismatchError("The number of function_calls and function_call_outputs must match.")
View Source 
var (
	ErrSpeechSchedulingPaused = errors.New("cannot schedule new speech, the speech scheduling is draining/pausing, the speech will be cancelled")
)

Functions

func ClientAgentStateString added in v0.1.0 

func ClientAgentStateString(state AgentState) (string, bool)

func ClientUserStateString added in v0.1.0 

func ClientUserStateString(state UserState) (string, bool)

func ContextWithAvatarStartInfo added in v0.0.15 

func ContextWithAvatarStartInfo(ctx context.Context, info AvatarStartInfo) context.Context

func MockTools added in v0.0.15 

func MockTools(ctx context.Context, agent AgentInterface, mocks map[string]MockToolFunc) context.Context

func PerformToolExecutions 

func PerformToolExecutions(
	ctx context.Context,
	functionCh <-chan *llm.FunctionToolCall,
	toolCtx *llm.ToolContext,
	opts ...ToolExecutionOption,
) <-chan ToolExecutionOutput

func UploadSessionReport 

func UploadSessionReport(
	cloudURL string,
	apiKey string,
	apiSecret string,
	agentName string,
	report *SessionReport,
) error

func WithRunContext 

func WithRunContext(ctx context.Context, rc *RunContext) context.Context

Types

type Agent 

type Agent struct {
	ID                  string
	Instructions        string
	InstructionVariants *llm.Instructions
	ChatCtx             *llm.ChatContext
	Tools               []llm.Tool

	TurnDetection     TurnDetectionMode
	TurnDetector      TurnDetector
	AudioTurnDetector AudioTurnDetector
	Avatar            AvatarProvider
	STT               stt.STT
	VAD               vad.VAD
	LLM               llm.LLM
	RealtimeModel     llm.RealtimeModel
	TTS               tts.TTS

	AllowInterruptions         bool
	AllowInterruptionsSet      bool
	MinConsecutiveSpeechDelay  float64
	UseTTSAlignedTranscript    bool
	UseTTSAlignedTranscriptSet bool
	MinEndpointingDelay        float64
	MaxEndpointingDelay        float64
	// contains filtered or unexported fields
}

func NewAgent 

func NewAgent(instructions string) *Agent

func (*Agent) ChatContext added in v0.0.15 

func (a *Agent) ChatContext() *llm.ChatContext

ChatContext returns a copy of the agent's current chat context.

This mirrors the reference SDK's read-only chat_ctx accessor: callers can inspect the returned context without mutating the agent's internal history. Use UpdateChatContext to replace the agent-owned context.

func (*Agent) GetActivity 

func (a *Agent) GetActivity() *AgentActivity

func (*Agent) GetAgent 

func (a *Agent) GetAgent() *Agent

func (*Agent) Label added in v0.0.15 

func (a *Agent) Label() string

func (*Agent) OnEnter 

func (a *Agent) OnEnter()

func (*Agent) OnExit 

func (a *Agent) OnExit()

func (*Agent) OnUserTurnCompleted 

func (a *Agent) OnUserTurnCompleted(ctx context.Context, chatCtx *llm.ChatContext, newMsg *llm.ChatMessage) error

func (*Agent) OnUserTurnExceeded added in v0.0.15 

func (a *Agent) OnUserTurnExceeded(ctx context.Context, ev UserTurnExceededEvent) error

func (*Agent) Start 

func (a *Agent) Start(session *AgentSession, agentIntf AgentInterface) *AgentActivity

func (*Agent) UpdateChatContext added in v0.0.15 

func (a *Agent) UpdateChatContext(ctx context.Context, chatCtx *llm.ChatContext, excludeInvalidFunctionCalls ...bool) error

func (*Agent) UpdateChatCtx added in v0.0.15 

func (a *Agent) UpdateChatCtx(ctx context.Context, chatCtx *llm.ChatContext, excludeInvalidFunctionCalls ...bool) error

func (*Agent) UpdateInstructions 

func (a *Agent) UpdateInstructions(ctx context.Context, instructions string) error

func (*Agent) UpdateTools 

func (a *Agent) UpdateTools(ctx context.Context, tools []llm.Tool) error

func (*Agent) UpdateTurnDetection added in v0.1.0 

func (a *Agent) UpdateTurnDetection(ctx context.Context, turnDetection TurnDetectionMode) error

type AgentActivity 

type AgentActivity struct {
	AgentIntf AgentInterface
	Agent     *Agent
	Session   *AgentSession
	// contains filtered or unexported fields
}

AgentActivity handles the internal event loops, I/O processing, and speech generation queue for an Agent.

func NewAgentActivity 

func NewAgentActivity(agentIntf AgentInterface, session *AgentSession) *AgentActivity

func (*AgentActivity) AllowInterruptions added in v0.0.15 

func (a *AgentActivity) AllowInterruptions() bool

func (*AgentActivity) ClearUserTurn added in v0.0.5 

func (a *AgentActivity) ClearUserTurn()

func (*AgentActivity) CommitUserTurn added in v0.0.5 

func (a *AgentActivity) CommitUserTurn(ctx context.Context, opts CommitUserTurnOptions) (string, error)

func (*AgentActivity) CurrentSpeech added in v0.0.15 

func (a *AgentActivity) CurrentSpeech() *SpeechHandle

func (*AgentActivity) Drain added in v0.0.5 

func (a *AgentActivity) Drain(ctx context.Context) error

func (*AgentActivity) EndpointingOpts added in v0.0.15 

func (a *AgentActivity) EndpointingOpts() EndpointingOptions

func (*AgentActivity) Interrupt added in v0.0.5 

func (a *AgentActivity) Interrupt(force bool) error

func (*AgentActivity) InterruptionEnabled added in v0.0.15 

func (a *AgentActivity) InterruptionEnabled() bool

func (*AgentActivity) MinConsecutiveSpeechDelay added in v0.0.15 

func (a *AgentActivity) MinConsecutiveSpeechDelay() time.Duration

func (*AgentActivity) OnEndOfSpeech 

func (a *AgentActivity) OnEndOfSpeech(ev *vad.VADEvent)

func (*AgentActivity) OnError added in v0.0.15 

func (a *AgentActivity) OnError(err error, source any)

func (*AgentActivity) OnFinalTranscript 

func (a *AgentActivity) OnFinalTranscript(ev *stt.SpeechEvent)

func (*AgentActivity) OnGenerationCreated added in v0.0.15 

func (a *AgentActivity) OnGenerationCreated(ev llm.GenerationCreatedEvent, configure ...func(*SpeechHandle)) (*SpeechHandle, error)

func (*AgentActivity) OnInputAudioTranscriptionCompleted added in v0.0.15 

func (a *AgentActivity) OnInputAudioTranscriptionCompleted(ev llm.InputTranscriptionCompleted)

func (*AgentActivity) OnInputSpeechStarted added in v0.0.15 

func (a *AgentActivity) OnInputSpeechStarted()

func (*AgentActivity) OnInputSpeechStopped added in v0.0.15 

func (a *AgentActivity) OnInputSpeechStopped(ev llm.InputSpeechStoppedEvent)

func (*AgentActivity) OnInterimTranscript added in v0.0.5 

func (a *AgentActivity) OnInterimTranscript(ev *stt.SpeechEvent)

func (*AgentActivity) OnInterruption added in v0.1.0 

func (a *AgentActivity) OnInterruption(ev OverlappingSpeechEvent)

func (*AgentActivity) OnMetricsCollected added in v0.0.15 

func (a *AgentActivity) OnMetricsCollected(metrics telemetry.AgentMetrics)

func (*AgentActivity) OnOverlapSpeechEnded added in v0.1.0 

func (a *AgentActivity) OnOverlapSpeechEnded(ev OverlappingSpeechEvent)

func (*AgentActivity) OnPipelineReplyDone added in v0.0.15 

func (a *AgentActivity) OnPipelineReplyDone(speech *SpeechHandle)

func (*AgentActivity) OnRemoteItemAdded added in v0.0.15 

func (a *AgentActivity) OnRemoteItemAdded(ev llm.RemoteItemAddedEvent)

func (*AgentActivity) OnSTTStartOfSpeech added in v0.1.0 

func (a *AgentActivity) OnSTTStartOfSpeech(ev *stt.SpeechEvent)

func (*AgentActivity) OnStartOfSpeech 

func (a *AgentActivity) OnStartOfSpeech(ev *vad.VADEvent)

Event callbacks from the active audio pipeline.

func (*AgentActivity) OnUserTurnExceeded added in v0.0.15 

func (a *AgentActivity) OnUserTurnExceeded(ev UserTurnExceededEvent)

func (*AgentActivity) OnVADInferenceDone added in v0.0.15 

func (a *AgentActivity) OnVADInferenceDone(ev *vad.VADEvent)

func (*AgentActivity) PauseScheduling added in v0.0.5 

func (a *AgentActivity) PauseScheduling()

func (*AgentActivity) RecordUserAudioFrame added in v0.1.0 

func (a *AgentActivity) RecordUserAudioFrame(frame *model.AudioFrame)

func (*AgentActivity) ResumeScheduling added in v0.0.5 

func (a *AgentActivity) ResumeScheduling()

func (*AgentActivity) RetrieveChatCtx added in v0.0.15 

func (a *AgentActivity) RetrieveChatCtx() *llm.ChatContext

func (*AgentActivity) ScheduleSpeech 

func (a *AgentActivity) ScheduleSpeech(speech *SpeechHandle, priority int, force bool) error

func (*AgentActivity) SchedulingPaused added in v0.0.15 

func (a *AgentActivity) SchedulingPaused() bool

func (*AgentActivity) Start 

func (a *AgentActivity) Start()

func (*AgentActivity) Stop 

func (a *AgentActivity) Stop()

func (*AgentActivity) Tools added in v0.0.15 

func (a *AgentActivity) Tools() []interface{}

func (*AgentActivity) UpdateChatContext added in v0.0.15 

func (a *AgentActivity) UpdateChatContext(ctx context.Context, chatCtx *llm.ChatContext, excludeInvalidFunctionCalls ...bool) error

func (*AgentActivity) UpdateChatCtx added in v0.0.15 

func (a *AgentActivity) UpdateChatCtx(ctx context.Context, chatCtx *llm.ChatContext, excludeInvalidFunctionCalls ...bool) error

func (*AgentActivity) UpdateInstructions added in v0.0.15 

func (a *AgentActivity) UpdateInstructions(ctx context.Context, instructions string) error

func (*AgentActivity) UpdateOptions added in v0.0.5 

func (a *AgentActivity) UpdateOptions(opts AgentSessionUpdateOptions) error

func (*AgentActivity) UpdateTools added in v0.0.15 

func (a *AgentActivity) UpdateTools(ctx context.Context, tools []llm.Tool) error

func (*AgentActivity) UseTTSAlignedTranscript added in v0.0.15 

func (a *AgentActivity) UseTTSAlignedTranscript() bool

func (*AgentActivity) WaitForInactive added in v0.0.15 

func (a *AgentActivity) WaitForInactive(ctx context.Context) error

type AgentEvent added in v0.0.5 

type AgentEvent = Event

type AgentFalseInterruptionEvent added in v0.0.5 

type AgentFalseInterruptionEvent struct {
	Resumed           bool
	CreatedAt         time.Time
	Message           *llm.ChatMessage
	ExtraInstructions string
}

func NewAgentFalseInterruptionEvent added in v0.0.15 

func NewAgentFalseInterruptionEvent(resumed bool) *AgentFalseInterruptionEvent

func (*AgentFalseInterruptionEvent) GetType added in v0.0.5 

func (e *AgentFalseInterruptionEvent) GetType() string

func (*AgentFalseInterruptionEvent) MarshalJSON added in v0.0.52 

func (e *AgentFalseInterruptionEvent) MarshalJSON() ([]byte, error)

type AgentHandoffEvent added in v0.0.5 

type AgentHandoffEvent struct {
	Item     *llm.AgentHandoff
	OldAgent *Agent
	NewAgent *Agent
}

func (*AgentHandoffEvent) GetCreatedAt added in v0.0.15 

func (e *AgentHandoffEvent) GetCreatedAt() time.Time

func (*AgentHandoffEvent) GetType added in v0.0.5 

func (e *AgentHandoffEvent) GetType() string

type AgentInterface 

type AgentInterface interface {
	OnEnter()
	OnExit()
	OnUserTurnCompleted(ctx context.Context, chatCtx *llm.ChatContext, newMsg *llm.ChatMessage) error
	OnUserTurnExceeded(ctx context.Context, ev UserTurnExceededEvent) error
	GetAgent() *Agent
	GetActivity() *AgentActivity
}

type AgentOutputTranscribedEvent added in v0.0.15 

type AgentOutputTranscribedEvent struct {
	Language   string
	Transcript string
	IsFinal    bool
	CreatedAt  time.Time
}

func (*AgentOutputTranscribedEvent) GetType added in v0.0.15 

func (e *AgentOutputTranscribedEvent) GetType() string

func (*AgentOutputTranscribedEvent) MarshalJSON added in v0.0.52 

func (e *AgentOutputTranscribedEvent) MarshalJSON() ([]byte, error)

type AgentReasoningTranscribedEvent added in v0.1.0 

type AgentReasoningTranscribedEvent struct {
	Text      string
	IsFinal   bool
	CreatedAt time.Time
}

func (*AgentReasoningTranscribedEvent) GetType added in v0.1.0 

func (e *AgentReasoningTranscribedEvent) GetType() string

func (*AgentReasoningTranscribedEvent) MarshalJSON added in v0.1.0 

func (e *AgentReasoningTranscribedEvent) MarshalJSON() ([]byte, error)

type AgentSession 

type AgentSession struct {
	Options AgentSessionOptions

	Timeline *EventTimeline

	ChatCtx       *llm.ChatContext
	Agent         AgentInterface
	STT           stt.STT
	VAD           vad.VAD
	LLM           llm.LLM
	TTS           tts.TTS
	RealtimeModel llm.RealtimeModel
	Tools         []llm.Tool
	Assistant     SessionAssistant
	Room          *lksdk.Room

	MetricsCollector    *telemetry.UsageCollector
	ModelUsageCollector *telemetry.ModelUsageCollector

	// UserTranscriptFilter, when non-nil, is applied before user transcript
	// events are recorded or broadcast to RoomIO subscribers.
	UserTranscriptFilter func(string) string

	// Event channels
	AgentStateChangedCh chan AgentStateChangedEvent
	UserStateChangedCh  chan UserStateChangedEvent
	// contains filtered or unexported fields
}

func NewAgentSession 

func NewAgentSession(agent AgentInterface, room *lksdk.Room, opts AgentSessionOptions) *AgentSession

func (*AgentSession) AgentFalseInterruptionEvents added in v0.0.15 

func (s *AgentSession) AgentFalseInterruptionEvents() <-chan AgentFalseInterruptionEvent

func (*AgentSession) AgentOutputTranscribedEvents added in v0.0.15 

func (s *AgentSession) AgentOutputTranscribedEvents() <-chan AgentOutputTranscribedEvent

func (*AgentSession) AgentReasoningTranscribedEvents added in v0.1.0 

func (s *AgentSession) AgentReasoningTranscribedEvents() <-chan AgentReasoningTranscribedEvent

func (*AgentSession) AgentState 

func (s *AgentSession) AgentState() AgentState

func (*AgentSession) AgentStateChangedEvents added in v0.0.15 

func (s *AgentSession) AgentStateChangedEvents() <-chan AgentStateChangedEvent

func (*AgentSession) AgentStateValue added in v0.0.15 

func (s *AgentSession) AgentStateValue() AgentState

func (*AgentSession) AudioOutputController added in v0.1.0 

func (s *AgentSession) AudioOutputController() AudioOutputController

func (*AgentSession) AudioPlaybackController added in v0.1.0 

func (s *AgentSession) AudioPlaybackController() AudioPlaybackController

func (*AgentSession) ClaimUserTurn added in v0.0.52 

func (s *AgentSession) ClaimUserTurn(ctx context.Context, fn func(context.Context) error) error

func (*AgentSession) ClearUserTurn added in v0.0.5 

func (s *AgentSession) ClearUserTurn() error

func (*AgentSession) CloseEvents added in v0.0.15 

func (s *AgentSession) CloseEvents() <-chan CloseEvent

func (*AgentSession) CloseSoon added in v0.0.15 

func (s *AgentSession) CloseSoon(reason CloseReason)

func (*AgentSession) CommitUserTurn added in v0.0.5 

func (s *AgentSession) CommitUserTurn(ctx context.Context, opts CommitUserTurnOptions) (string, error)

func (*AgentSession) ConversationItemAddedEvents added in v0.0.15 

func (s *AgentSession) ConversationItemAddedEvents() <-chan ConversationItemAddedEvent

func (*AgentSession) CurrentAgent added in v0.0.15 

func (s *AgentSession) CurrentAgent() (AgentInterface, error)

func (*AgentSession) CurrentSpeech added in v0.0.15 

func (s *AgentSession) CurrentSpeech() *SpeechHandle

func (*AgentSession) Drain added in v0.0.15 

func (s *AgentSession) Drain(ctx context.Context) error

func (*AgentSession) EmitAgentFalseInterruption added in v0.0.15 

func (s *AgentSession) EmitAgentFalseInterruption(ev AgentFalseInterruptionEvent)

func (*AgentSession) EmitAgentOutputTranscribed added in v0.0.15 

func (s *AgentSession) EmitAgentOutputTranscribed(ev AgentOutputTranscribedEvent)

func (*AgentSession) EmitAgentReasoningTranscribed added in v0.1.0 

func (s *AgentSession) EmitAgentReasoningTranscribed(ev AgentReasoningTranscribedEvent)

func (*AgentSession) EmitConversationItemAdded added in v0.0.15 

func (s *AgentSession) EmitConversationItemAdded(item llm.ChatItem)

func (*AgentSession) EmitError added in v0.0.15 

func (s *AgentSession) EmitError(ev ErrorEvent)

func (*AgentSession) EmitEvent added in v0.1.6 

func (s *AgentSession) EmitEvent(ev Event)

func (*AgentSession) EmitFunctionToolsExecuted added in v0.0.15 

func (s *AgentSession) EmitFunctionToolsExecuted(ev FunctionToolsExecutedEvent) FunctionToolsExecutedEvent

func (*AgentSession) EmitMetricsCollected added in v0.0.15 

func (s *AgentSession) EmitMetricsCollected(metrics telemetry.AgentMetrics)

func (*AgentSession) EmitOverlappingSpeech added in v0.0.15 

func (s *AgentSession) EmitOverlappingSpeech(ev OverlappingSpeechEvent)

func (*AgentSession) EmitSessionUsageUpdated added in v0.0.15 

func (s *AgentSession) EmitSessionUsageUpdated(ev SessionUsageUpdatedEvent)

func (*AgentSession) EmitSipDTMF added in v0.0.15 

func (s *AgentSession) EmitSipDTMF(ev SipDTMFEvent)

func (*AgentSession) EmitSpeechCreated added in v0.0.15 

func (s *AgentSession) EmitSpeechCreated(ev SpeechCreatedEvent)

func (*AgentSession) EmitUserInputTranscribed added in v0.0.15 

func (s *AgentSession) EmitUserInputTranscribed(ev UserInputTranscribedEvent)

func (*AgentSession) EmitUserTurnExceeded added in v0.0.15 

func (s *AgentSession) EmitUserTurnExceeded(ev UserTurnExceededEvent)

func (*AgentSession) EnsureAssistant added in v0.0.15 

func (s *AgentSession) EnsureAssistant() SessionAssistant

func (*AgentSession) ErrorEvents added in v0.0.15 

func (s *AgentSession) ErrorEvents() <-chan ErrorEvent

func (*AgentSession) FunctionToolsExecutedEvents added in v0.0.15 

func (s *AgentSession) FunctionToolsExecutedEvents() <-chan FunctionToolsExecutedEvent

func (*AgentSession) GenerateReply 

func (s *AgentSession) GenerateReply(ctx context.Context, userInput string) (*SpeechHandle, error)

func (*AgentSession) GenerateReplyWithOptions added in v0.0.15 

func (s *AgentSession) GenerateReplyWithOptions(ctx context.Context, opts GenerateReplyOptions) (*SpeechHandle, error)

func (*AgentSession) History added in v0.0.15 

func (s *AgentSession) History() *llm.ChatContext

func (*AgentSession) Interrupt added in v0.0.5 

func (s *AgentSession) Interrupt(force bool) error

func (*AgentSession) JobContext added in v0.0.15 

func (s *AgentSession) JobContext() (any, error)

func (*AgentSession) MCPServers added in v0.0.15 

func (s *AgentSession) MCPServers() []llm.MCPServer

func (*AgentSession) MetricsCollectedEvents added in v0.0.15 

func (s *AgentSession) MetricsCollectedEvents() <-chan MetricsCollectedEvent

func (*AgentSession) ModelUsage added in v0.0.15 

func (s *AgentSession) ModelUsage() telemetry.AgentSessionUsage

func (*AgentSession) Off added in v0.0.15 

func (s *AgentSession) Off(eventType string, callback func(Event))

func (*AgentSession) On added in v0.0.15 

func (s *AgentSession) On(eventType string, callback func(Event)) func()

func (*AgentSession) OnAudioEnabledChanged added in v0.0.15 

func (s *AgentSession) OnAudioEnabledChanged(enabled bool)

func (*AgentSession) OnAudioFrame 

func (s *AgentSession) OnAudioFrame(ctx context.Context, frame *model.AudioFrame)

func (*AgentSession) OnVideoFrame added in v0.0.15 

func (s *AgentSession) OnVideoFrame(ctx context.Context, frame *images.VideoFrame)

func (*AgentSession) Once added in v0.0.15 

func (s *AgentSession) Once(eventType string, callback func(Event)) func()

func (*AgentSession) OverlappingSpeechEvents added in v0.0.15 

func (s *AgentSession) OverlappingSpeechEvents() <-chan OverlappingSpeechEvent

func (*AgentSession) RecordedEvents added in v0.0.15 

func (s *AgentSession) RecordedEvents() []Event

func (*AgentSession) RefreshUserAwayTimer added in v0.1.0 

func (s *AgentSession) RefreshUserAwayTimer()

func (*AgentSession) Run added in v0.0.15 

func (s *AgentSession) Run(ctx context.Context, userInput string) (*RunResult, error)

func (*AgentSession) RunWithOptions added in v0.0.15 

func (s *AgentSession) RunWithOptions(ctx context.Context, opts RunOptions) (*RunResult, error)

func (*AgentSession) Say added in v0.0.5 

func (s *AgentSession) Say(ctx context.Context, text string) (*SpeechHandle, error)

func (*AgentSession) SayWithOptions added in v0.0.15 

func (s *AgentSession) SayWithOptions(ctx context.Context, opts SayOptions) (*SpeechHandle, error)

func (*AgentSession) SessionOptions added in v0.0.15 

func (s *AgentSession) SessionOptions() AgentSessionOptions

func (*AgentSession) SessionUsageUpdatedEvents added in v0.0.15 

func (s *AgentSession) SessionUsageUpdatedEvents() <-chan SessionUsageUpdatedEvent

func (*AgentSession) SetAudioOutputController added in v0.1.0 

func (s *AgentSession) SetAudioOutputController(controller AudioOutputController)

func (*AgentSession) SetAudioPlaybackController added in v0.1.0 

func (s *AgentSession) SetAudioPlaybackController(controller AudioPlaybackController)

func (*AgentSession) SetJobContext added in v0.0.15 

func (s *AgentSession) SetJobContext(value any)

func (*AgentSession) SetMCPServers added in v0.0.15 

func (s *AgentSession) SetMCPServers(servers []llm.MCPServer)

func (*AgentSession) SetUserAwayTimerGate added in v0.1.0 

func (s *AgentSession) SetUserAwayTimerGate(gate func() bool)

SetUserAwayTimerGate installs a runtime gate for user-away timer arming. The callback returns true while the timer should be blocked.

func (*AgentSession) SetUserdata added in v0.0.15 

func (s *AgentSession) SetUserdata(value any)

func (*AgentSession) Shutdown added in v0.0.15 

func (s *AgentSession) Shutdown(drain ...bool)

func (*AgentSession) SipDTMFEvents added in v0.0.15 

func (s *AgentSession) SipDTMFEvents() <-chan SipDTMFEvent

func (*AgentSession) SpeechCreatedEvents added in v0.0.15 

func (s *AgentSession) SpeechCreatedEvents() <-chan SpeechCreatedEvent

func (*AgentSession) Start 

func (s *AgentSession) Start(ctx context.Context) error

func (*AgentSession) StartWithOptions added in v0.0.15 

func (s *AgentSession) StartWithOptions(ctx context.Context, opts StartOptions) (*RunResult, error)

func (*AgentSession) Stop 

func (s *AgentSession) Stop(ctx context.Context) error

func (*AgentSession) TurnDetection added in v0.0.15 

func (s *AgentSession) TurnDetection() TurnDetectionMode

func (*AgentSession) UpdateAgent added in v0.0.5 

func (s *AgentSession) UpdateAgent(agent AgentInterface)

func (*AgentSession) UpdateAgentState 

func (s *AgentSession) UpdateAgentState(state AgentState)

func (*AgentSession) UpdateOptions added in v0.0.5 

func (s *AgentSession) UpdateOptions(opts AgentSessionUpdateOptions) error

func (*AgentSession) UpdateUserState 

func (s *AgentSession) UpdateUserState(state UserState)

func (*AgentSession) Usage added in v0.0.15 

func (s *AgentSession) Usage() telemetry.UsageSummary

func (*AgentSession) UserInputTranscribedEvents added in v0.0.15 

func (s *AgentSession) UserInputTranscribedEvents() <-chan UserInputTranscribedEvent

func (*AgentSession) UserState 

func (s *AgentSession) UserState() UserState

func (*AgentSession) UserStateChangedEvents added in v0.0.15 

func (s *AgentSession) UserStateChangedEvents() <-chan UserStateChangedEvent

func (*AgentSession) UserStateValue added in v0.0.15 

func (s *AgentSession) UserStateValue() UserState

func (*AgentSession) UserTurnExceededEvents added in v0.0.15 

func (s *AgentSession) UserTurnExceededEvents() <-chan UserTurnExceededEvent

func (*AgentSession) Userdata added in v0.0.15 

func (s *AgentSession) Userdata() (any, error)

func (*AgentSession) WaitForInactive added in v0.0.15 

func (s *AgentSession) WaitForInactive(ctx context.Context) error

func (*AgentSession) WaitForInactiveAndHold added in v0.0.52 

func (s *AgentSession) WaitForInactiveAndHold(ctx context.Context, fn func(context.Context) error) error

type AgentSessionOptions 

type AgentSessionOptions struct {
	AllowInterruptions                       bool
	AllowInterruptionsSet                    bool
	DiscardAudioIfUninterruptible            bool
	DiscardAudioIfUninterruptibleSet         bool
	MinInterruptionDuration                  float64
	MinInterruptionDurationSet               bool
	MinInterruptionWords                     int
	MinEndpointingDelay                      float64
	MinEndpointingDelaySet                   bool
	MaxEndpointingDelay                      float64
	MaxEndpointingDelaySet                   bool
	EndpointingMode                          string
	EndpointingAlpha                         float64
	Endpointing                              Endpointing
	MaxToolSteps                             int
	MaxToolStepsSet                          bool
	UserAwayTimeout                          float64
	UserAwayTimeoutSet                       bool
	DisableUserAwayTimeout                   bool
	UserTurnLimitMaxWords                    int
	UserTurnLimitMaxDuration                 float64
	FalseInterruptionTimeout                 float64
	FalseInterruptionTimeoutSet              bool
	ResumeFalseInterruption                  bool
	ResumeFalseInterruptionSet               bool
	BackchannelBoundaryStart                 float64
	BackchannelBoundaryStartSet              bool
	BackchannelBoundaryEnd                   float64
	BackchannelBoundaryEndSet                bool
	MinConsecutiveSpeechDelay                float64
	UseTTSAlignedTranscript                  bool
	TTSStreamPacer                           *tts.SentenceStreamPacerOptions
	TTSTextReplacements                      map[string]string
	TTSTextTransforms                        []string
	TTSTextTransformsSet                     bool
	DisableTTSTextTransforms                 bool
	LLMParallelToolCalls                     *bool
	LLMExtraParams                           map[string]any
	LLMResponseFormat                        map[string]any
	BackgroundAudio                          *BackgroundAudioPlayer
	WordTokenizer                            tokenize.WordTokenizer
	PreemptiveGeneration                     bool
	PreemptiveGenerationSet                  bool
	PreemptiveGenerationPreemptiveTTS        bool
	PreemptiveGenerationPreemptiveTTSSet     bool
	PreemptiveGenerationMaxSpeechDuration    float64
	PreemptiveGenerationMaxSpeechDurationSet bool
	PreemptiveGenerationMaxRetries           int
	PreemptiveGenerationMaxRetriesSet        bool
	AudioInputHook                           AudioInputHook
	AECWarmupDuration                        float64
	AECWarmupDurationSet                     bool
	SessionCloseTranscriptTimeout            float64
	SessionCloseTranscriptTimeoutSet         bool
	TurnDetection                            TurnDetectionMode
	IVRDetection                             bool
	IVRSilenceDuration                       time.Duration
	VideoSampler                             *VoiceActivityVideoSampler
	ToolChoice                               llm.ToolChoice
	MaxUnrecoverableErrors                   int
	MaxUnrecoverableErrorsSet                bool
	MockTools                                map[string]MockToolFunc
}

type AgentSessionUpdateOptions added in v0.0.15 

type AgentSessionUpdateOptions struct {
	MinEndpointingDelay *float64
	MaxEndpointingDelay *float64
	EndpointingAlpha    *float64
	EndpointingMode     *string
	Endpointing         Endpointing
	TurnDetection       *TurnDetectionMode
	ToolChoice          *llm.ToolChoice
}

type AgentState 

type AgentState string

type AgentStateChangedEvent 

type AgentStateChangedEvent struct {
	OldState  AgentState
	NewState  AgentState
	CreatedAt time.Time
}

func (*AgentStateChangedEvent) GetType added in v0.0.5 

func (e *AgentStateChangedEvent) GetType() string

func (*AgentStateChangedEvent) MarshalJSON added in v0.0.52 

func (e *AgentStateChangedEvent) MarshalJSON() ([]byte, error)

type AgentTask 

type AgentTask[T any] struct {
	Agent
	Result chan T
	Err    chan error
	// contains filtered or unexported fields
}

AgentTask represents a sub-agent execution that returns a result

func NewAgentTask 

func NewAgentTask[T any](instructions string) *AgentTask[T]

func (*AgentTask[T]) Cancel added in v0.0.15 

func (t *AgentTask[T]) Cancel()

func (*AgentTask[T]) Complete 

func (t *AgentTask[T]) Complete(result T) error

func (*AgentTask[T]) Done added in v0.0.15 

func (t *AgentTask[T]) Done() bool

func (*AgentTask[T]) Fail 

func (t *AgentTask[T]) Fail(err error) error

func (*AgentTask[T]) Wait added in v0.0.15 

func (t *AgentTask[T]) Wait(ctx context.Context) (T, error)

func (*AgentTask[T]) WaitAny 

func (t *AgentTask[T]) WaitAny(ctx context.Context) (any, error)

type AudioConfig 

type AudioConfig struct {
	Source      AudioSource
	Volume      float64
	Probability float64
	FadeIn      float64
	FadeOut     float64
}

type AudioInputHook added in v0.0.15 

type AudioInputHook func(ctx context.Context, frame *model.AudioFrame) *model.AudioFrame

AudioInputHook is called for each incoming audio frame before it is forwarded to VAD and STT processing. Returning nil drops the frame from both pipelines.

type AudioOutputController added in v0.1.0 

type AudioOutputController interface {
	CanPauseAudioOutput() bool
	PauseAudioOutput()
	ResumeAudioOutput()
}

type AudioPlaybackController added in v0.1.0 

type AudioPlaybackController interface {
	ClearBuffer()
	WaitForPlayout(context.Context) (AudioPlaybackResult, error)
}

type AudioPlaybackResult added in v0.1.0 

type AudioPlaybackResult struct {
	PlaybackPosition          time.Duration
	Interrupted               bool
	SynchronizedTranscript    string
	HasSynchronizedTranscript bool
}

type AudioSource 

type AudioSource interface{} // Can be string, BuiltinAudioClip, or <-chan *model.AudioFrame

type AudioTurnDetector added in v0.1.0 

type AudioTurnDetector interface {
	PredictEndOfTurnAudio(ctx context.Context, frames []*model.AudioFrame) (float64, error)
}

type AvatarMetricsHandler added in v0.0.15 

type AvatarMetricsHandler func(*telemetry.AvatarMetrics)

type AvatarMetricsSource added in v0.0.15 

type AvatarMetricsSource interface {
	OnMetricsCollected(handler AvatarMetricsHandler) func()
}

type AvatarProvider added in v0.0.15 

type AvatarProvider interface {
	Start(ctx context.Context) error
	UpdateState(state AvatarState) error
}

type AvatarStartInfo added in v0.0.15 

type AvatarStartInfo struct {
	LiveKitURL    string
	LiveKitToken  string
	RoomName      string
	AgentIdentity string
}

func AvatarStartInfoFromContext added in v0.0.15 

func AvatarStartInfoFromContext(ctx context.Context) (AvatarStartInfo, bool)

type AvatarState 

type AvatarState string
const (
	AvatarStateIdle     AvatarState = "idle"
	AvatarStateSpeaking AvatarState = "speaking"
)

type BackgroundAudioPlayer 

type BackgroundAudioPlayer struct {
	// contains filtered or unexported fields
}

func NewBackgroundAudioPlayer 

func NewBackgroundAudioPlayer(ambientSound, thinkingSound interface{}) *BackgroundAudioPlayer

func (*BackgroundAudioPlayer) AgentStateChanged 

func (p *BackgroundAudioPlayer) AgentStateChanged(newState AgentState)

func (*BackgroundAudioPlayer) Close 

func (p *BackgroundAudioPlayer) Close() error

func (*BackgroundAudioPlayer) Play 

func (p *BackgroundAudioPlayer) Play(audio interface{}, loop bool) *PlayHandle

func (*BackgroundAudioPlayer) Start 

func (p *BackgroundAudioPlayer) Start(room *lksdk.Room, agentSession *AgentSession) error

type BaseEndpointing added in v0.0.15 

type BaseEndpointing struct {
	// contains filtered or unexported fields
}

func NewBaseEndpointing added in v0.0.15 

func NewBaseEndpointing(minDelay float64, maxDelay float64) *BaseEndpointing

func (*BaseEndpointing) MaxDelay added in v0.0.15 

func (e *BaseEndpointing) MaxDelay() float64

func (*BaseEndpointing) MinDelay added in v0.0.15 

func (e *BaseEndpointing) MinDelay() float64

func (*BaseEndpointing) OnEndOfAgentSpeech added in v0.0.15 

func (e *BaseEndpointing) OnEndOfAgentSpeech(endedAt float64)

func (*BaseEndpointing) OnEndOfSpeech added in v0.0.15 

func (e *BaseEndpointing) OnEndOfSpeech(endedAt float64, shouldIgnore bool)

func (*BaseEndpointing) OnStartOfAgentSpeech added in v0.0.15 

func (e *BaseEndpointing) OnStartOfAgentSpeech(startedAt float64)

func (*BaseEndpointing) OnStartOfSpeech added in v0.0.15 

func (e *BaseEndpointing) OnStartOfSpeech(startedAt float64, overlapping bool)

func (*BaseEndpointing) Overlapping added in v0.0.15 

func (e *BaseEndpointing) Overlapping() bool

func (*BaseEndpointing) UpdateOptions added in v0.0.15 

func (e *BaseEndpointing) UpdateOptions(minDelay *float64, maxDelay *float64)

type BuiltinAudioClip 

type BuiltinAudioClip string
const (
	CityAmbience    BuiltinAudioClip = "city-ambience.ogg"
	ForestAmbience  BuiltinAudioClip = "forest-ambience.ogg"
	OfficeAmbience  BuiltinAudioClip = "office-ambience.ogg"
	CrowdedRoom     BuiltinAudioClip = "crowded-room.ogg"
	KeyboardTyping  BuiltinAudioClip = "keyboard-typing.ogg"
	KeyboardTyping2 BuiltinAudioClip = "keyboard-typing2.ogg"
	HoldMusic       BuiltinAudioClip = "hold_music.ogg"
)

func (BuiltinAudioClip) Path 

func (b BuiltinAudioClip) Path() string

type ChatMessageEvent added in v0.0.15 

type ChatMessageEvent struct {
	Item *llm.ChatMessage
}

func (*ChatMessageEvent) GetCreatedAt added in v0.0.15 

func (e *ChatMessageEvent) GetCreatedAt() time.Time

func (*ChatMessageEvent) GetType added in v0.0.15 

func (e *ChatMessageEvent) GetType() string

type CloseEvent 

type CloseEvent struct {
	Reason    CloseReason
	Error     error
	CreatedAt time.Time
}

func (*CloseEvent) GetType 

func (e *CloseEvent) GetType() string

func (*CloseEvent) MarshalJSON added in v0.0.52 

func (e *CloseEvent) MarshalJSON() ([]byte, error)

type CloseReason 

type CloseReason string
const (
	CloseReasonError                   CloseReason = "error"
	CloseReasonJobShutdown             CloseReason = "job_shutdown"
	CloseReasonParticipantDisconnected CloseReason = "participant_disconnected"
	CloseReasonUserInitiated           CloseReason = "user_initiated"
	CloseReasonTaskCompleted           CloseReason = "task_completed"
)

type CommitUserTurnOptions added in v0.0.15 

type CommitUserTurnOptions struct {
	TranscriptTimeout time.Duration
	STTFlushDuration  time.Duration
	SkipReply         bool
	SkipRealtimeAudio bool
}

type ConversationItemAddedEvent 

type ConversationItemAddedEvent struct {
	Item      llm.ChatItem
	CreatedAt time.Time
}

func (*ConversationItemAddedEvent) GetType 

func (e *ConversationItemAddedEvent) GetType() string

func (*ConversationItemAddedEvent) MarshalJSON added in v0.0.52 

func (e *ConversationItemAddedEvent) MarshalJSON() ([]byte, error)

type DynamicEndpointing added in v0.0.15 

type DynamicEndpointing struct {
	BaseEndpointing
	// contains filtered or unexported fields
}

func NewDynamicEndpointing added in v0.0.15 

func NewDynamicEndpointing(minDelay float64, maxDelay float64, alpha ...float64) *DynamicEndpointing

func (*DynamicEndpointing) BetweenTurnDelay added in v0.0.15 

func (e *DynamicEndpointing) BetweenTurnDelay() float64

func (*DynamicEndpointing) BetweenUtteranceDelay added in v0.0.15 

func (e *DynamicEndpointing) BetweenUtteranceDelay() float64

func (*DynamicEndpointing) ImmediateInterruptionDelay added in v0.0.15 

func (e *DynamicEndpointing) ImmediateInterruptionDelay() (float64, float64)

func (*DynamicEndpointing) MaxDelay added in v0.0.15 

func (e *DynamicEndpointing) MaxDelay() float64

func (*DynamicEndpointing) MinDelay added in v0.0.15 

func (e *DynamicEndpointing) MinDelay() float64

func (*DynamicEndpointing) OnEndOfAgentSpeech added in v0.0.15 

func (e *DynamicEndpointing) OnEndOfAgentSpeech(endedAt float64)

func (*DynamicEndpointing) OnEndOfSpeech added in v0.0.15 

func (e *DynamicEndpointing) OnEndOfSpeech(endedAt float64, shouldIgnore bool)

func (*DynamicEndpointing) OnStartOfAgentSpeech added in v0.0.15 

func (e *DynamicEndpointing) OnStartOfAgentSpeech(startedAt float64)

func (*DynamicEndpointing) OnStartOfSpeech added in v0.0.15 

func (e *DynamicEndpointing) OnStartOfSpeech(startedAt float64, overlapping bool)

func (*DynamicEndpointing) UpdateAlpha added in v0.0.15 

func (e *DynamicEndpointing) UpdateAlpha(alpha float64)

func (*DynamicEndpointing) UpdateOptions added in v0.0.15 

func (e *DynamicEndpointing) UpdateOptions(minDelay *float64, maxDelay *float64)

type EndOfTurnInfo 

type EndOfTurnInfo struct {
	SkipReply              bool
	ReplyAlreadyGenerated  bool
	GenerateReplyAfterHook bool
	NewTranscript          string
	Language               string
	TranscriptConfidence   float64
	EndOfTurnDelay         float64
	TranscriptionDelay     float64
	StartedSpeakingAt      *float64
	StoppedSpeakingAt      *float64
	AudioFrames            []*model.AudioFrame
}

type Endpointing added in v0.0.15 

type Endpointing interface {
	UpdateOptions(minDelay *float64, maxDelay *float64)
	MinDelay() float64
	MaxDelay() float64
	Overlapping() bool
	OnStartOfSpeech(startedAt float64, overlapping bool)
	OnEndOfSpeech(endedAt float64, shouldIgnore bool)
	OnStartOfAgentSpeech(startedAt float64)
	OnEndOfAgentSpeech(endedAt float64)
}

func CreateEndpointing added in v0.0.15 

func CreateEndpointing(kind string, minDelay float64, maxDelay float64, alpha ...float64) Endpointing

type EndpointingOptions added in v0.0.15 

type EndpointingOptions struct {
	Mode     string
	MinDelay float64
	MaxDelay float64
}

type ErrorEvent added in v0.0.5 

type ErrorEvent struct {
	Error     error
	Source    any
	CreatedAt time.Time
}

func NewErrorEvent added in v0.0.15 

func NewErrorEvent(err error, source any) *ErrorEvent

func (*ErrorEvent) GetType added in v0.0.5 

func (e *ErrorEvent) GetType() string

func (*ErrorEvent) MarshalJSON added in v0.0.52 

func (e *ErrorEvent) MarshalJSON() ([]byte, error)

type EvaluationResult 

type EvaluationResult struct {
	Judgments    map[string]string
	Reasoning    map[string]string
	Instructions map[string]string
}

type Event 

type Event interface {
	GetType() string
}

type EventTimeline added in v0.0.5 

type EventTimeline struct {
	// contains filtered or unexported fields
}

func NewEventTimeline added in v0.0.5 

func NewEventTimeline(session *AgentSession) *EventTimeline

func (*EventTimeline) AddEvent added in v0.0.5 

func (t *EventTimeline) AddEvent(ev Event)

func (*EventTimeline) AddSubscriber added in v0.0.13 

func (t *EventTimeline) AddSubscriber(callback func(Event)) func()

type FillerOptions added in v0.1.0 

type FillerOptions struct {
	Text         string
	Source       func(step int) (string, bool)
	SpeechSource func(step int) (*SpeechHandle, bool)
	Delay        time.Duration
	Interval     *time.Duration
	MaxSteps     *int
}

type FunctionCallEvent added in v0.0.15 

type FunctionCallEvent struct {
	Item *llm.FunctionCall
}

func (*FunctionCallEvent) GetCreatedAt added in v0.0.15 

func (e *FunctionCallEvent) GetCreatedAt() time.Time

func (*FunctionCallEvent) GetType added in v0.0.15 

func (e *FunctionCallEvent) GetType() string

type FunctionCallOutputEvent added in v0.0.15 

type FunctionCallOutputEvent struct {
	Item *llm.FunctionCallOutput
}

func (*FunctionCallOutputEvent) GetCreatedAt added in v0.0.15 

func (e *FunctionCallOutputEvent) GetCreatedAt() time.Time

func (*FunctionCallOutputEvent) GetType added in v0.0.15 

func (e *FunctionCallOutputEvent) GetType() string

type FunctionToolExecutionPair added in v0.0.15 

type FunctionToolExecutionPair struct {
	FunctionCall       *llm.FunctionCall
	FunctionCallOutput *llm.FunctionCallOutput
}

type FunctionToolsExecutedEvent added in v0.0.5 

type FunctionToolsExecutedEvent struct {
	FunctionCalls       []*llm.FunctionCall
	FunctionCallOutputs []*llm.FunctionCallOutput
	CreatedAt           time.Time
	ReplyRequired       bool
	HandoffRequired     bool
}

func NewFunctionToolsExecutedEvent added in v0.0.15 

func NewFunctionToolsExecutedEvent(calls []*llm.FunctionCall, outputs []*llm.FunctionCallOutput) (*FunctionToolsExecutedEvent, error)

func (*FunctionToolsExecutedEvent) CancelAgentHandoff added in v0.0.15 

func (e *FunctionToolsExecutedEvent) CancelAgentHandoff()

func (*FunctionToolsExecutedEvent) CancelToolReply added in v0.0.15 

func (e *FunctionToolsExecutedEvent) CancelToolReply()

func (*FunctionToolsExecutedEvent) GetType added in v0.0.5 

func (e *FunctionToolsExecutedEvent) GetType() string

func (*FunctionToolsExecutedEvent) HasAgentHandoff added in v0.0.5 

func (e *FunctionToolsExecutedEvent) HasAgentHandoff() bool

func (*FunctionToolsExecutedEvent) HasToolReply added in v0.0.5 

func (e *FunctionToolsExecutedEvent) HasToolReply() bool

func (*FunctionToolsExecutedEvent) MarshalJSON added in v0.0.52 

func (e *FunctionToolsExecutedEvent) MarshalJSON() ([]byte, error)

func (*FunctionToolsExecutedEvent) Zipped added in v0.0.15 

func (e *FunctionToolsExecutedEvent) Zipped() []FunctionToolExecutionPair

type GenerateReplyOptions added in v0.0.15 

type GenerateReplyOptions struct {
	UserInput           string
	UserMessage         *llm.ChatMessage
	Instructions        string
	InstructionVariants *llm.Instructions
	ToolChoice          llm.ToolChoice
	Tools               []string
	ChatCtx             *llm.ChatContext
	AllowInterruptions  *bool
	InputModality       string
	ScheduleSpeech      *bool
	UserInitiated       *bool
}

type IVRActivity 

type IVRActivity struct {
	Session *AgentSession
	// contains filtered or unexported fields
}

func NewIVRActivity 

func NewIVRActivity(session *AgentSession) *IVRActivity

func (*IVRActivity) Start 

func (i *IVRActivity) Start()

func (*IVRActivity) Stop 

func (i *IVRActivity) Stop()

type InputDetails 

type InputDetails struct {
	Modality string
}

func DefaultInputDetails 

func DefaultInputDetails() InputDetails

type LLMGenerationData 

type LLMGenerationData struct {
	TextCh             chan string
	TextEventCh        chan LLMTextEvent
	FunctionCh         chan *llm.FunctionToolCall
	Done               chan struct{}
	GeneratedText      string
	GeneratedFunctions []llm.FunctionToolCall
	GeneratedExtra     map[string]any
	ID                 string
	TTFT               time.Duration
	Duration           time.Duration
	RequestID          string
	Usage              *llm.CompletionUsage
	StreamErr          error
}

func PerformLLMInference 

func PerformLLMInference(
	ctx context.Context,
	l llm.LLM,
	chatCtx *llm.ChatContext,
	tools []llm.Tool,
	options ...llm.ChatOption,
) (*LLMGenerationData, error)

func PerformLLMInferenceWithTextEvents added in v0.1.0 

func PerformLLMInferenceWithTextEvents(
	ctx context.Context,
	l llm.LLM,
	chatCtx *llm.ChatContext,
	tools []llm.Tool,
	options ...llm.ChatOption,
) (*LLMGenerationData, error)

type LLMTextEvent added in v0.1.0 

type LLMTextEvent struct {
	Text  string
	Flush bool
}

type LLMTurnDetector 

type LLMTurnDetector struct {
	// contains filtered or unexported fields
}

LLMTurnDetector uses an LLM to predict if the user has finished speaking. It sends the recent conversation history to the LLM and asks for a probability score.

func NewLLMTurnDetector 

func NewLLMTurnDetector(llmInstance llm.LLM) *LLMTurnDetector

func (*LLMTurnDetector) PredictEndOfTurn 

func (m *LLMTurnDetector) PredictEndOfTurn(ctx context.Context, chatCtx *llm.ChatContext) (float64, error)

type MetricsCollectedEvent 

type MetricsCollectedEvent struct {
	Metrics   telemetry.AgentMetrics
	CreatedAt time.Time
}

func (*MetricsCollectedEvent) GetType 

func (e *MetricsCollectedEvent) GetType() string

func (*MetricsCollectedEvent) MarshalJSON added in v0.0.52 

func (e *MetricsCollectedEvent) MarshalJSON() ([]byte, error)

type MockToolFunc added in v0.0.15 

type MockToolFunc func(ctx context.Context, args string) (string, error)

type MultimodalAgent 

type MultimodalAgent struct {
	PublishAudio func(ctx context.Context, frame *model.AudioFrame) error
	// contains filtered or unexported fields
}

func NewMultimodalAgent 

func NewMultimodalAgent(
	m llm.RealtimeModel,
	chatCtx *llm.ChatContext,
) *MultimodalAgent

func (*MultimodalAgent) ClearAudio added in v0.0.18 

func (ma *MultimodalAgent) ClearAudio() error

func (*MultimodalAgent) Close added in v0.0.15 

func (ma *MultimodalAgent) Close() error

func (*MultimodalAgent) CommitAudio added in v0.0.18 

func (ma *MultimodalAgent) CommitAudio() error

func (*MultimodalAgent) Interrupt added in v0.0.18 

func (ma *MultimodalAgent) Interrupt() error

func (*MultimodalAgent) OnAudioFrame 

func (ma *MultimodalAgent) OnAudioFrame(ctx context.Context, frame *model.AudioFrame)

func (*MultimodalAgent) OnSpeechScheduled added in v0.0.15 

func (ma *MultimodalAgent) OnSpeechScheduled(ctx context.Context, speech *SpeechHandle)

func (*MultimodalAgent) OnVideoFrame added in v0.0.15 

func (ma *MultimodalAgent) OnVideoFrame(ctx context.Context, frame *images.VideoFrame)

func (*MultimodalAgent) RealtimeCapabilities added in v0.0.15 

func (ma *MultimodalAgent) RealtimeCapabilities() llm.RealtimeCapabilities

func (*MultimodalAgent) SetPublishAudio added in v0.0.15 

func (ma *MultimodalAgent) SetPublishAudio(publish func(ctx context.Context, frame *model.AudioFrame) error)

func (*MultimodalAgent) Start 

func (ma *MultimodalAgent) Start(ctx context.Context, s *AgentSession) error

func (*MultimodalAgent) SupportsNativeSay added in v0.0.15 

func (ma *MultimodalAgent) SupportsNativeSay() bool

func (*MultimodalAgent) UpdateChatContext added in v0.0.15 

func (ma *MultimodalAgent) UpdateChatContext(ctx context.Context, chatCtx *llm.ChatContext) error

func (*MultimodalAgent) UpdateInstructions added in v0.0.15 

func (ma *MultimodalAgent) UpdateInstructions(ctx context.Context, instructions string) error

func (*MultimodalAgent) UpdateOptions added in v0.0.15 

func (ma *MultimodalAgent) UpdateOptions(ctx context.Context, options llm.RealtimeSessionOptions) error

func (*MultimodalAgent) UpdateRealtimeModel added in v0.0.15 

func (ma *MultimodalAgent) UpdateRealtimeModel(ctx context.Context, model llm.RealtimeModel) error

func (*MultimodalAgent) UpdateTools added in v0.0.15 

func (ma *MultimodalAgent) UpdateTools(ctx context.Context) error

type OverlappingSpeechEvent added in v0.0.15 

type OverlappingSpeechEvent struct {
	CreatedAt          time.Time
	DetectedAt         time.Time
	IsInterruption     bool
	TotalDuration      time.Duration
	PredictionDuration time.Duration
	DetectionDelay     time.Duration
	OverlapStartedAt   *time.Time
	SpeechInput        []int16
	Probabilities      []float32
	Probability        float64
	NumRequests        int
}

func (*OverlappingSpeechEvent) GetType added in v0.0.15 

func (e *OverlappingSpeechEvent) GetType() string

func (*OverlappingSpeechEvent) MarshalJSON added in v0.0.52 

func (e *OverlappingSpeechEvent) MarshalJSON() ([]byte, error)

type PipelineAgent 

type PipelineAgent struct {
	LLM llm.LLM

	PublishAudio func(ctx context.Context, frame *model.AudioFrame) error
	// contains filtered or unexported fields
}

func NewPipelineAgent 

func NewPipelineAgent(
	vad vad.VAD,
	stt stt.STT,
	llmObj llm.LLM,
	tts tts.TTS,
	chatCtx *llm.ChatContext,
) *PipelineAgent

func (*PipelineAgent) ClearInputTranscription added in v0.1.0 

func (va *PipelineAgent) ClearInputTranscription() error

func (*PipelineAgent) FlushInputTranscription added in v0.1.0 

func (va *PipelineAgent) FlushInputTranscription(ctx context.Context, duration time.Duration) error

func (*PipelineAgent) OnAudioFrame 

func (va *PipelineAgent) OnAudioFrame(ctx context.Context, frame *model.AudioFrame)

func (*PipelineAgent) OnSpeechPreemptive added in v0.1.0 

func (va *PipelineAgent) OnSpeechPreemptive(ctx context.Context, speech *SpeechHandle)

func (*PipelineAgent) OnSpeechScheduled added in v0.0.15 

func (va *PipelineAgent) OnSpeechScheduled(ctx context.Context, speech *SpeechHandle)

func (*PipelineAgent) SetAudioInputHook added in v0.0.15 

func (va *PipelineAgent) SetAudioInputHook(hook AudioInputHook)

SetAudioInputHook registers a hook that intercepts incoming audio before VAD and STT. Passing nil clears the hook.

func (*PipelineAgent) SetPublishAudio added in v0.0.15 

func (va *PipelineAgent) SetPublishAudio(publish func(ctx context.Context, frame *model.AudioFrame) error)

func (*PipelineAgent) Start 

func (va *PipelineAgent) Start(ctx context.Context, s *AgentSession) error

func (*PipelineAgent) UpdateComponents added in v0.0.15 

func (va *PipelineAgent) UpdateComponents(vadObj vad.VAD, sttObj stt.STT, llmObj llm.LLM, ttsObj tts.TTS)

type PlayHandle 

type PlayHandle struct {
	// contains filtered or unexported fields
}

func (*PlayHandle) Done 

func (h *PlayHandle) Done() bool

func (*PlayHandle) Stop 

func (h *PlayHandle) Stop()

func (*PlayHandle) WaitForPlayout 

func (h *PlayHandle) WaitForPlayout()

type RecordingOptions 

type RecordingOptions struct {
	Audio      bool `json:"audio"`
	Traces     bool `json:"traces"`
	Logs       bool `json:"logs"`
	Transcript bool `json:"transcript"`
}

type RunAssert 

type RunAssert struct {
	ChatCtx *llm.ChatContext
	// contains filtered or unexported fields
}

func (*RunAssert) ContainsAgentHandoff added in v0.0.15 

func (a *RunAssert) ContainsAgentHandoff(newAgent *Agent) *RunAssert

func (*RunAssert) ContainsEventInRange added in v0.0.15 

func (a *RunAssert) ContainsEventInRange(start int, end int, eventType string, criteria ...RunEventCriteria) *RunAssert

func (*RunAssert) ContainsFunctionCall added in v0.0.15 

func (a *RunAssert) ContainsFunctionCall(name string) *RunAssert

func (*RunAssert) ContainsFunctionCallMatching added in v0.0.15 

func (a *RunAssert) ContainsFunctionCallMatching(criteria RunEventCriteria) *RunAssert

func (*RunAssert) ContainsFunctionCallOutput added in v0.0.15 

func (a *RunAssert) ContainsFunctionCallOutput(output string, isError bool) *RunAssert

func (*RunAssert) ContainsFunctionCallOutputMatching added in v0.0.15 

func (a *RunAssert) ContainsFunctionCallOutputMatching(criteria RunEventCriteria) *RunAssert

func (*RunAssert) ContainsFunctionCallWithArguments added in v0.0.15 

func (a *RunAssert) ContainsFunctionCallWithArguments(name string, arguments map[string]any) *RunAssert

func (*RunAssert) ContainsMessage 

func (a *RunAssert) ContainsMessage(role llm.ChatRole, content string) *RunAssert

func (*RunAssert) ContainsMessageMatching added in v0.0.15 

func (a *RunAssert) ContainsMessageMatching(criteria RunEventCriteria) *RunAssert

func (*RunAssert) ContainsMessageRole added in v0.0.15 

func (a *RunAssert) ContainsMessageRole(role llm.ChatRole) *RunAssert

func (*RunAssert) EventAt added in v0.0.15 

func (a *RunAssert) EventAt(index int, eventType string, criteria ...RunEventCriteria) *RunAssert

func (*RunAssert) HasError 

func (a *RunAssert) HasError() error

func (*RunAssert) IsFunctionCall 

func (a *RunAssert) IsFunctionCall(name string) *RunAssert

func (*RunAssert) IsFunctionCallWithArguments added in v0.0.15 

func (a *RunAssert) IsFunctionCallWithArguments(name string, arguments map[string]any) *RunAssert

func (*RunAssert) Judge 

func (a *RunAssert) Judge(ctx context.Context, evaluator evals.Evaluator, llmInstance llm.LLM) (*RunAssert, error)

func (*RunAssert) NextAgentHandoff added in v0.0.15 

func (a *RunAssert) NextAgentHandoff(newAgent *Agent) *RunAssert

func (*RunAssert) NextEvent added in v0.0.15 

func (a *RunAssert) NextEvent(eventType ...string) *RunAssert

func (*RunAssert) NextFunctionCall added in v0.0.15 

func (a *RunAssert) NextFunctionCall(name string) *RunAssert

func (*RunAssert) NextFunctionCallOutput added in v0.0.15 

func (a *RunAssert) NextFunctionCallOutput(criteria RunEventCriteria) *RunAssert

func (*RunAssert) NextFunctionCallWithArguments added in v0.0.15 

func (a *RunAssert) NextFunctionCallWithArguments(name string, arguments map[string]any) *RunAssert

func (*RunAssert) NextMessage added in v0.0.15 

func (a *RunAssert) NextMessage(role llm.ChatRole) *RunAssert

func (*RunAssert) NoMoreEvents added in v0.0.15 

func (a *RunAssert) NoMoreEvents() *RunAssert

func (*RunAssert) SkipNext added in v0.0.15 

func (a *RunAssert) SkipNext(count int) *RunAssert

func (*RunAssert) SkipNextEventIf added in v0.0.15 

func (a *RunAssert) SkipNextEventIf(eventType string, criteria ...RunEventCriteria) bool

type RunContext 

type RunContext struct {
	Session      *AgentSession
	SpeechHandle *SpeechHandle
	FunctionCall *llm.FunctionCall
	// contains filtered or unexported fields
}

func GetRunContext 

func GetRunContext(ctx context.Context) *RunContext

func NewRunContext added in v0.0.15 

func NewRunContext(session *AgentSession, speechHandle *SpeechHandle, functionCall *llm.FunctionCall) *RunContext

func (*RunContext) DisallowInterruptions added in v0.0.15 

func (r *RunContext) DisallowInterruptions() error

func (*RunContext) EmittedUpdateCount added in v0.1.0 

func (r *RunContext) EmittedUpdateCount() int

func (*RunContext) Foreground added in v0.0.52 

func (r *RunContext) Foreground(ctx context.Context, fn func(context.Context) error) error

func (*RunContext) JobContext added in v0.0.15 

func (r *RunContext) JobContext() (any, error)

func (*RunContext) Update added in v0.0.52 

func (r *RunContext) Update(message any, templates ...string) error

func (*RunContext) Updates added in v0.0.52 

func (r *RunContext) Updates() []RunContextUpdate

func (*RunContext) Userdata added in v0.0.15 

func (r *RunContext) Userdata() (any, error)

func (*RunContext) WaitForPlayout 

func (r *RunContext) WaitForPlayout(ctx context.Context) error

func (*RunContext) WithFiller added in v0.1.0 

func (r *RunContext) WithFiller(ctx context.Context, opts FillerOptions, fn func(context.Context) error) error

type RunContextUpdate added in v0.0.52 

type RunContextUpdate struct {
	FunctionCall       *llm.FunctionCall
	FunctionCallOutput *llm.FunctionCallOutput
}

type RunEvent added in v0.0.5 

type RunEvent interface {
	GetType() string
	GetCreatedAt() time.Time
}

type RunEventCriteria added in v0.0.15 

type RunEventCriteria struct {
	Role      llm.ChatRole
	Name      string
	Arguments map[string]any
	Output    string
	OutputSet bool
	IsError   *bool
	NewAgent  *Agent
}

type RunOptions added in v0.0.15 

type RunOptions struct {
	UserInput          string
	AllowInterruptions *bool
	InputModality      string
	OutputType         reflect.Type
}

type RunResult 

type RunResult struct {
	ChatCtx *llm.ChatContext
	Expect  *RunAssert
	// contains filtered or unexported fields
}

func NewRunResult 

func NewRunResult(chatCtx *llm.ChatContext) *RunResult

func NewRunResultWithOutputType added in v0.0.15 

func NewRunResultWithOutputType(chatCtx *llm.ChatContext, outputType reflect.Type) *RunResult

func NewRunResultWithUserInput added in v0.0.15 

func NewRunResultWithUserInput(chatCtx *llm.ChatContext, userInput string) *RunResult

func (*RunResult) Done added in v0.0.5 

func (r *RunResult) Done() bool

func (*RunResult) Events added in v0.0.5 

func (r *RunResult) Events() []RunEvent

func (*RunResult) FinalOutput added in v0.0.5 

func (r *RunResult) FinalOutput() (any, error)

func (*RunResult) MarkDone added in v0.0.15 

func (r *RunResult) MarkDone()

func (*RunResult) RecordAgentHandoff added in v0.0.15 

func (r *RunResult) RecordAgentHandoff(item *llm.AgentHandoff, oldAgent *Agent, newAgent *Agent)

func (*RunResult) RecordItem added in v0.0.15 

func (r *RunResult) RecordItem(item llm.ChatItem)

func (*RunResult) SetFinalOutput added in v0.0.15 

func (r *RunResult) SetFinalOutput(output any)

func (*RunResult) UnwatchSpeechHandle added in v0.0.15 

func (r *RunResult) UnwatchSpeechHandle(speech *SpeechHandle) bool

func (*RunResult) UserInput added in v0.0.15 

func (r *RunResult) UserInput() string

func (*RunResult) Wait added in v0.0.5 

func (r *RunResult) Wait(ctx context.Context) error

func (*RunResult) WatchSpeechHandle added in v0.0.15 

func (r *RunResult) WatchSpeechHandle(speech *SpeechHandle) bool

type SayOptions added in v0.0.15 

type SayOptions struct {
	Text               string
	AllowInterruptions *bool
	AddToChatContext   *bool
}

type SessionAssistant added in v0.0.15 

type SessionAssistant interface {
	Start(ctx context.Context, s *AgentSession) error
	OnAudioFrame(ctx context.Context, frame *model.AudioFrame)
	SetPublishAudio(func(ctx context.Context, frame *model.AudioFrame) error)
}

type SessionReport 

type SessionReport struct {
	RecordingOptions        RecordingOptions        `json:"recording_options"`
	JobID                   string                  `json:"job_id"`
	RoomID                  string                  `json:"room_id"`
	Room                    string                  `json:"room"`
	Options                 AgentSessionOptions     `json:"options"`
	Events                  []any                   `json:"events"`
	ChatHistory             *llm.ChatContext        `json:"chat_history"`
	AudioRecordingPath      *string                 `json:"audio_recording_path,omitempty"`
	AudioRecordingStartedAt *float64                `json:"audio_recording_started_at,omitempty"`
	Duration                *float64                `json:"duration,omitempty"`
	StartedAt               *float64                `json:"started_at,omitempty"`
	LLMModel                string                  `json:"llm_model,omitempty"`
	LLMProvider             string                  `json:"llm_provider,omitempty"`
	RealtimeModel           string                  `json:"realtime_model,omitempty"`
	RealtimeProvider        string                  `json:"realtime_provider,omitempty"`
	Timestamp               float64                 `json:"timestamp"`
	Usage                   *telemetry.UsageSummary `json:"usage,omitempty"`
	ModelUsage              []telemetry.ModelUsage  `json:"model_usage,omitempty"`
	SDKVersion              string                  `json:"sdk_version"`
	Tagger                  *Tagger                 `json:"-"`
}

func NewSessionReport 

func NewSessionReport(sessions ...*AgentSession) *SessionReport

func (*SessionReport) MarshalJSON added in v0.0.15 

func (r *SessionReport) MarshalJSON() ([]byte, error)

func (*SessionReport) ToDict added in v0.0.5 

func (r *SessionReport) ToDict() map[string]any

type SessionUsageUpdatedEvent added in v0.0.15 

type SessionUsageUpdatedEvent struct {
	Usage     telemetry.AgentSessionUsage
	CreatedAt time.Time
}

func (*SessionUsageUpdatedEvent) GetType added in v0.0.15 

func (e *SessionUsageUpdatedEvent) GetType() string

func (*SessionUsageUpdatedEvent) MarshalJSON added in v0.0.52 

func (e *SessionUsageUpdatedEvent) MarshalJSON() ([]byte, error)

type SipDTMFEvent added in v0.0.15 

type SipDTMFEvent struct {
	Digit          string
	Code           uint32
	SenderIdentity string
}

type SpeechCreatedEvent 

type SpeechCreatedEvent struct {
	UserInitiated bool
	Source        string // "say" or "generate_reply"
	SpeechHandle  *SpeechHandle
	CreatedAt     time.Time
}

func (*SpeechCreatedEvent) GetType 

func (e *SpeechCreatedEvent) GetType() string

func (*SpeechCreatedEvent) MarshalJSON added in v0.0.52 

func (e *SpeechCreatedEvent) MarshalJSON() ([]byte, error)

type SpeechGenerationOptions added in v0.0.15 

type SpeechGenerationOptions struct {
	Instructions       *llm.Instructions
	ToolChoice         llm.ToolChoice
	Tools              []string
	IgnoreOnEnterTools bool
	ChatCtx            *llm.ChatContext
	UserMessage        *llm.ChatMessage
	AssistantMessage   *llm.ChatMessage
	Text               string
	RealtimeGeneration *llm.GenerationCreatedEvent
}

type SpeechHandle 

type SpeechHandle struct {
	ID                 string
	AllowInterruptions bool
	InputDetails       InputDetails
	Generation         SpeechGenerationOptions
	Priority           int
	CreatedAt          time.Time
	// contains filtered or unexported fields
}

func NewSpeechHandle 

func NewSpeechHandle(allowInterruptions bool, inputDetails InputDetails) *SpeechHandle

func (*SpeechHandle) AddChatItems added in v0.0.15 

func (s *SpeechHandle) AddChatItems(items ...llm.ChatItem)

func (*SpeechHandle) AddDoneCallback added in v0.0.15 

func (s *SpeechHandle) AddDoneCallback(callback func(*SpeechHandle)) func()

func (*SpeechHandle) AddItemAddedCallback added in v0.0.15 

func (s *SpeechHandle) AddItemAddedCallback(callback func(llm.ChatItem)) func()

func (*SpeechHandle) AllowsInterruptions added in v0.1.0 

func (s *SpeechHandle) AllowsInterruptions() bool

func (*SpeechHandle) AuthorizeGeneration added in v0.0.15 

func (s *SpeechHandle) AuthorizeGeneration()

func (*SpeechHandle) ChatItems added in v0.0.15 

func (s *SpeechHandle) ChatItems() []llm.ChatItem

func (*SpeechHandle) ClearAuthorization added in v0.0.15 

func (s *SpeechHandle) ClearAuthorization()

func (*SpeechHandle) DoneSequence added in v0.1.0 

func (s *SpeechHandle) DoneSequence() uint64

func (*SpeechHandle) GenerationID added in v0.0.15 

func (s *SpeechHandle) GenerationID() string

func (*SpeechHandle) IncrementStep added in v0.0.15 

func (s *SpeechHandle) IncrementStep()

func (*SpeechHandle) Interrupt 

func (s *SpeechHandle) Interrupt(force bool) error

func (*SpeechHandle) IsDone 

func (s *SpeechHandle) IsDone() bool

func (*SpeechHandle) IsInterrupted 

func (s *SpeechHandle) IsInterrupted() bool

func (*SpeechHandle) IsScheduled 

func (s *SpeechHandle) IsScheduled() bool

func (*SpeechHandle) MarkDone 

func (s *SpeechHandle) MarkDone()

func (*SpeechHandle) MarkGenerationDone added in v0.0.15 

func (s *SpeechHandle) MarkGenerationDone() error

func (*SpeechHandle) MarkScheduled 

func (s *SpeechHandle) MarkScheduled()

func (*SpeechHandle) NumSteps added in v0.0.15 

func (s *SpeechHandle) NumSteps() int

func (*SpeechHandle) ParentGenerationID added in v0.0.15 

func (s *SpeechHandle) ParentGenerationID() string

func (*SpeechHandle) RunFinalOutput added in v0.0.15 

func (s *SpeechHandle) RunFinalOutput() (any, bool)

func (*SpeechHandle) SetAllowInterruptions added in v0.0.15 

func (s *SpeechHandle) SetAllowInterruptions(allow bool) error

func (*SpeechHandle) SetRunFinalOutput added in v0.0.15 

func (s *SpeechHandle) SetRunFinalOutput(output any)

func (*SpeechHandle) Wait 

func (s *SpeechHandle) Wait(ctx context.Context) error

func (*SpeechHandle) WaitForAuthorization added in v0.0.15 

func (s *SpeechHandle) WaitForAuthorization(ctx context.Context) error

func (*SpeechHandle) WaitForGeneration added in v0.0.15 

func (s *SpeechHandle) WaitForGeneration(ctx context.Context, stepIndex int) error

func (*SpeechHandle) WaitForScheduled added in v0.0.15 

func (s *SpeechHandle) WaitForScheduled(ctx context.Context) error

func (*SpeechHandle) WaitIfNotInterrupted added in v0.0.15 

func (s *SpeechHandle) WaitIfNotInterrupted(ctx context.Context, workDone ...<-chan error) error

type StartOptions added in v0.0.15 

type StartOptions struct {
	CaptureRun bool
	OutputType reflect.Type
}

type TTSGenerationData 

type TTSGenerationData struct {
	AudioCh        chan *model.AudioFrame
	TimedTextCh    chan tts.TimedString
	TTFB           time.Duration
	StreamErr      error
	ForwardedAudio bool
}

func PerformTTSInference 

func PerformTTSInference(ctx context.Context, t tts.TTS, textCh <-chan string, opts ...TTSInferenceOption) (*TTSGenerationData, error)

type TTSInferenceOption added in v0.0.15 

type TTSInferenceOption func(*TTSInferenceOptions)

func WithOrderedTTSTextReplacements added in v0.1.0 

func WithOrderedTTSTextReplacements(replacements []tts.TextReplacement) TTSInferenceOption

func WithTTSPreserveTimedTranscript added in v0.0.15 

func WithTTSPreserveTimedTranscript() TTSInferenceOption

func WithTTSRequireAudio added in v0.1.0 

func WithTTSRequireAudio() TTSInferenceOption

func WithTTSStreamPacer added in v0.0.15 

func WithTTSStreamPacer(opts tts.SentenceStreamPacerOptions) TTSInferenceOption

func WithTTSTextReplacements added in v0.0.15 

func WithTTSTextReplacements(replacements map[string]string) TTSInferenceOption

func WithTTSTextTransforms added in v0.1.0 

func WithTTSTextTransforms(transforms []string) TTSInferenceOption

func WithTTSTextTransformsDisabled added in v0.0.15 

func WithTTSTextTransformsDisabled() TTSInferenceOption

type TTSInferenceOptions added in v0.0.15 

type TTSInferenceOptions struct {
	StreamPacer             *tts.SentenceStreamPacerOptions
	TextReplacements        map[string]string
	OrderedTextReplacements []tts.TextReplacement
	TextTransforms          []string
	TextTransformsSet       bool
	DisableTextTransforms   bool
	PreserveTimedTranscript bool
	RequireAudio            bool
}

type TagMetadata added in v0.0.18 

type TagMetadata struct {
	Name      string
	Metadata  map[string]any
	Timestamp time.Time
}

type Tagger 

type Tagger struct {
	// contains filtered or unexported fields
}

func NewTagger 

func NewTagger() *Tagger

func (*Tagger) Add 

func (t *Tagger) Add(tag string, metadata ...map[string]any)

func (*Tagger) Evaluation 

func (t *Tagger) Evaluation(result *EvaluationResult)

func (*Tagger) Evaluations added in v0.0.15 

func (t *Tagger) Evaluations() []map[string]any

func (*Tagger) Fail 

func (t *Tagger) Fail(reason string)

func (*Tagger) MetadataTags added in v0.0.18 

func (t *Tagger) MetadataTags() []TagMetadata

func (*Tagger) Outcome added in v0.0.15 

func (t *Tagger) Outcome() string

func (*Tagger) OutcomeReason 

func (t *Tagger) OutcomeReason() string

func (*Tagger) Remove 

func (t *Tagger) Remove(tag string)

func (*Tagger) Success 

func (t *Tagger) Success(reason string)

func (*Tagger) Tags 

func (t *Tagger) Tags() []string

type TaskWaiter 

type TaskWaiter interface {
	WaitAny(ctx context.Context) (any, error)
}

type ToolExecutionOption added in v0.0.15 

type ToolExecutionOption func(*ToolExecutionOptions)

func WithToolExecutionSession added in v0.0.15 

func WithToolExecutionSession(session *AgentSession) ToolExecutionOption

func WithToolExecutionSpeechHandle added in v0.0.15 

func WithToolExecutionSpeechHandle(speechHandle *SpeechHandle) ToolExecutionOption

func WithToolExecutionToolChoice added in v0.0.15 

func WithToolExecutionToolChoice(toolChoice llm.ToolChoice) ToolExecutionOption

type ToolExecutionOptions added in v0.0.15 

type ToolExecutionOptions struct {
	Session      *AgentSession
	SpeechHandle *SpeechHandle
	ToolChoice   llm.ToolChoice
}

type ToolExecutionOutput 

type ToolExecutionOutput struct {
	FncCall              llm.FunctionCall
	FncCallOut           *llm.FunctionCallOutput
	RawOutput            any
	RawError             error
	RunContextUpdate     bool
	RunContextUpdateDone chan struct{}
}

type TranscriptSynchronizer 

type TranscriptSynchronizer struct {
	// contains filtered or unexported fields
}

TranscriptSynchronizer drip-feeds text to match the playout speed of audio.

func NewTranscriptSynchronizer 

func NewTranscriptSynchronizer(speakingRate float64) *TranscriptSynchronizer

NewTranscriptSynchronizer initializes the synchronizer. Default speaking rate is usually ~3.83 syllables/sec.

func (*TranscriptSynchronizer) Close 

func (s *TranscriptSynchronizer) Close()

func (*TranscriptSynchronizer) EventCh 

func (s *TranscriptSynchronizer) EventCh() <-chan string

func (*TranscriptSynchronizer) Interrupt 

func (s *TranscriptSynchronizer) Interrupt()

Interrupt immediately flushes the remaining text buffer to the event channel and stops syncing.

func (*TranscriptSynchronizer) PushAudio 

func (s *TranscriptSynchronizer) PushAudio(frame *model.AudioFrame)

func (*TranscriptSynchronizer) PushText 

func (s *TranscriptSynchronizer) PushText(text string)

type TurnDetectionMode 

type TurnDetectionMode string
const (
	TurnDetectionModeSTT         TurnDetectionMode = "stt"
	TurnDetectionModeVAD         TurnDetectionMode = "vad"
	TurnDetectionModeRealtimeLLM TurnDetectionMode = "realtime_llm"
	TurnDetectionModeManual      TurnDetectionMode = "manual"
)

type TurnDetector 

type TurnDetector interface {
	PredictEndOfTurn(ctx context.Context, chatCtx *llm.ChatContext) (float64, error)
}

type UserInputTranscribedEvent 

type UserInputTranscribedEvent struct {
	Language   string
	Transcript string
	IsFinal    bool
	SpeakerID  string
	CreatedAt  time.Time
}

func (*UserInputTranscribedEvent) GetType 

func (e *UserInputTranscribedEvent) GetType() string

func (*UserInputTranscribedEvent) MarshalJSON added in v0.0.52 

func (e *UserInputTranscribedEvent) MarshalJSON() ([]byte, error)

type UserState 

type UserState string

type UserStateChangedEvent 

type UserStateChangedEvent struct {
	OldState  UserState
	NewState  UserState
	CreatedAt time.Time
}

func (*UserStateChangedEvent) GetType added in v0.0.5 

func (e *UserStateChangedEvent) GetType() string

func (*UserStateChangedEvent) MarshalJSON added in v0.0.52 

func (e *UserStateChangedEvent) MarshalJSON() ([]byte, error)

type UserTurnExceededEvent added in v0.0.15 

type UserTurnExceededEvent struct {
	Transcript            string
	AccumulatedTranscript string
	AccumulatedWordCount  int
	Duration              time.Duration
	CreatedAt             time.Time
}

func NewUserTurnExceededEvent added in v0.0.15 

func NewUserTurnExceededEvent(transcript string, accumulatedTranscript string, accumulatedWordCount int, duration time.Duration) *UserTurnExceededEvent

func (*UserTurnExceededEvent) GetType added in v0.0.15 

func (e *UserTurnExceededEvent) GetType() string

func (*UserTurnExceededEvent) MarshalJSON added in v0.0.52 

func (e *UserTurnExceededEvent) MarshalJSON() ([]byte, error)

type VoiceActivityVideoSampler 

type VoiceActivityVideoSampler struct {
	// contains filtered or unexported fields
}

VoiceActivityVideoSampler samples video frames at a reduced rate (e.g. 1 fps) only when the user is speaking, to reduce LLM context token usage.

func NewVoiceActivityVideoSampler 

func NewVoiceActivityVideoSampler(session *AgentSession, sampleRate float64, opts images.EncodeOptions) *VoiceActivityVideoSampler

func NewVoiceActivityVideoSamplerWithRates added in v0.0.15 

func NewVoiceActivityVideoSamplerWithRates(session *AgentSession, speakingFPS float64, silentFPS float64, opts images.EncodeOptions) *VoiceActivityVideoSampler

func (*VoiceActivityVideoSampler) OnVideoFrame 

func (s *VoiceActivityVideoSampler) OnVideoFrame(ctx context.Context, frame *images.VideoFrame) bool

OnVideoFrame should be called for every incoming WebRTC video frame. It returns true if the frame should be forwarded to the LLM.

func (*VoiceActivityVideoSampler) SetSpeaking 

func (s *VoiceActivityVideoSampler) SetSpeaking(speaking bool)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.