frames

type AdapterType ¶ added in v0.1.0

type AdapterType string

AdapterType names the wire format a custom tool is written in. Several providers share one: everything speaking OpenAI's API reads a tool written for OpenAI, whether it arrives over chat completions, the Responses API or a realtime session.

const (
	// AdapterTypeOpenAI is OpenAI's tool format: chat completions, the Responses
	// API and the Realtime API alike.
	AdapterTypeOpenAI AdapterType = "openai"
	// AdapterTypeGemini is Gemini's tool format.
	AdapterTypeGemini AdapterType = "gemini"
)

type AggregatedTextFrame ¶ added in v0.1.0

type AggregatedTextFrame struct {
	TextFrame
	// AggregatedBy is how the text was aggregated.
	AggregatedBy AggregationType
	// ContextID identifies the synthesis context this text was sent on.
	ContextID string
	// RawText is the full matched text including any pattern delimiters, set
	// when the frame came from a pattern match such as a code block. Empty for
	// an ordinary sentence aggregation.
	RawText string
	// WillBeSpoken reports whether the TTS service will speak this frame. The
	// service sets it just before synthesis.
	WillBeSpoken bool
}

AggregatedTextFrame is a run of TextFrames aggregated into one unit for synthesis, carrying how they were aggregated.

func NewAggregatedTextFrame(text string, by AggregationType) *AggregatedTextFrame

func (f *AggregatedTextFrame) String() string

type AggregatedTextProgressFrame ¶ added in v0.1.0

type AggregatedTextProgressFrame struct {
	BaseDataFrame
	// SegmentID identifies the AggregatedTextFrame being spoken.
	SegmentID uint64
	// ContextID is the synthesis context the text belongs to.
	ContextID string
	// Text is the full original text of the frame being spoken.
	Text string
	// AggregatedBy is how that text was aggregated.
	AggregatedBy AggregationType
	// AccumulatedText is what has been spoken so far, including the current word.
	AccumulatedText string
	// RemainingText is what has not been spoken yet.
	RemainingText string
}

AggregatedTextProgressFrame accompanies each TTSTextFrame during word-timestamp playback, carrying the spoken-so-far and remaining text of the AggregatedTextFrame being spoken. It lets a consumer highlight words as they are heard without reaching into the sequencer's state.

func NewAggregatedTextProgressFrame(segmentID uint64, contextID, text string,
	by AggregationType, accumulated, remaining string,
) *AggregatedTextProgressFrame

func (f *AggregatedTextProgressFrame) String() string

type AggregationType ¶ added in v0.1.0

type AggregationType string

AggregationType names how a stream of text was aggregated before synthesis.

const (
	// AggregationSentence aggregates text up to a sentence boundary.
	AggregationSentence AggregationType = "sentence"
	// AggregationToken passes each token through as it arrives.
	AggregationToken AggregationType = "token"
	// AggregationWord aggregates text a word at a time.
	AggregationWord AggregationType = "word"
	// AnyAggregation is not a way of aggregating text but a wildcard standing for
	// every one of them, for something registered against a type that should
	// apply whatever the unit was grouped by.
	AnyAggregation AggregationType = "*"
)

type AsyncToolKind ¶ added in v0.1.0

type AsyncToolKind string

AsyncToolKind identifies which stage of the async-tool protocol a message is.

const (
	// AsyncToolStarted marks the message appended when the tool starts running.
	AsyncToolStarted AsyncToolKind = "started"
	// AsyncToolIntermediate marks an intermediate result reported while the tool
	// is still running.
	AsyncToolIntermediate AsyncToolKind = "intermediate"
	// AsyncToolFinal marks the tool's final result.
	AsyncToolFinal AsyncToolKind = "final"
)

type AsyncToolMessage ¶ added in v0.1.0

type AsyncToolMessage struct {
	// Kind is which of the three stages this message is.
	Kind AsyncToolKind
	// ToolCallID is the id of the tool invocation the message relates to.
	ToolCallID string
	// Status is "running" for started and intermediate messages, "finished" for
	// the final one.
	Status string
	// Description is the human-readable description carried in the payload. It
	// may be empty.
	Description string
	// Result is the result string for intermediate and final messages, and empty
	// for a started message. HasResult distinguishes the two.
	Result string
	// HasResult reports whether the payload carried a result field at all.
	HasResult bool
}

AsyncToolMessage is the structured contents of an async-tool message.

func ParseAsyncToolMessage(m Message) (AsyncToolMessage, bool)

type AudioBufferStartRecordingFrame ¶ added in v0.0.4

type AudioBufferStartRecordingFrame struct {
	BaseControlFrame
	UninterruptibleMixin
}

AudioBufferStartRecordingFrame instructs audio-buffer processors to start recording. It is a control frame and is uninterruptible so a barge-in does not drop it.

func NewAudioBufferStartRecordingFrame() *AudioBufferStartRecordingFrame

type AudioBufferStopRecordingFrame ¶ added in v0.0.4

type AudioBufferStopRecordingFrame struct {
	BaseControlFrame
	UninterruptibleMixin
}

AudioBufferStopRecordingFrame instructs audio-buffer processors to stop recording and flush the buffered audio. It is a control frame and is uninterruptible.

func NewAudioBufferStopRecordingFrame() *AudioBufferStopRecordingFrame

type AudioFrame ¶ added in v0.1.0

type AudioFrame interface {
	Frame
	AudioData() *AudioRawData
}

AudioFrame is implemented by every frame that carries raw audio, whichever direction it travels in.

type AudioRawData ¶

type AudioRawData struct {
	// Audio is raw PCM audio: 16-bit signed samples, interleaved by channel.
	Audio []byte
	// SampleRate is the audio sample rate in Hz.
	SampleRate int
	// NumChannels is the number of interleaved audio channels.
	NumChannels int
}

AudioRawData is the raw-audio payload shared by the audio frame types. It carries a chunk of PCM audio plus the metadata needed to interpret it; the audio frames embed it alongside a category base.

func (a *AudioRawData) AudioData() *AudioRawData

func (a *AudioRawData) NumFrames() int

type AutoSummarizationConfig ¶ added in v0.1.0

type AutoSummarizationConfig struct {
	// MaxContextTokens is the estimated context size above which the
	// conversation is compressed. Nil turns the token threshold off, leaving
	// MaxUnsummarizedMessages the only trigger.
	MaxContextTokens *int
	// MaxUnsummarizedMessages is how many messages may accumulate since the last
	// summary before the conversation is compressed again, so it is compressed
	// regularly even when the token threshold is never reached. Nil turns the
	// message threshold off.
	MaxUnsummarizedMessages *int
	// SummaryConfig controls how the summary itself is generated.
	SummaryConfig SummaryConfig
}

AutoSummarizationConfig controls when a conversation is compressed automatically, and how the summary it produces is generated. Summarization runs when either threshold is passed.

Each threshold is a pointer so that turning it off is distinct from leaving it at its default: nil disables that threshold, and the two may not both be nil. Build one with NewAutoSummarizationConfig to start from the defaults.

func NewAutoSummarizationConfig() AutoSummarizationConfig

func (c AutoSummarizationConfig) Validate() error

func (c AutoSummarizationConfig) WithDefaults() AutoSummarizationConfig

type BaseControlFrame ¶

type BaseControlFrame struct{ BaseFrame }

BaseControlFrame is embedded by control frames. Construct with NewBaseControlFrame.

func NewBaseControlFrame(typeName string) BaseControlFrame

type BaseDataFrame ¶

type BaseDataFrame struct{ BaseFrame }

BaseDataFrame is embedded by data frames. Construct with NewBaseDataFrame.

func NewBaseDataFrame(typeName string) BaseDataFrame

type BaseFrame ¶

type BaseFrame struct {
	// contains filtered or unexported fields
}

BaseFrame is embedded by every concrete frame and implements Frame. Construct it with NewBaseFrame so the id and name are initialized.

func NewBaseFrame(typeName string) BaseFrame

func (f *BaseFrame) Base() *BaseFrame

func (f *BaseFrame) BroadcastSiblingID() (uint64, bool)

func (f *BaseFrame) ID() uint64

func (f *BaseFrame) Metadata() map[string]any

func (f *BaseFrame) Name() string

func (f *BaseFrame) PTS() (int64, bool)

func (f *BaseFrame) SetBroadcastSiblingID(id uint64)

func (f *BaseFrame) SetPTS(pts int64)

func (f *BaseFrame) SetTransportDestination(dest string)

func (f *BaseFrame) SetTransportSource(source string)

func (f *BaseFrame) String() string

func (f *BaseFrame) TransportDestination() string

func (f *BaseFrame) TransportSource() string

type BaseMetricsData ¶ added in v0.1.0

type BaseMetricsData struct {
	// Processor is the name of the processor that measured it.
	Processor string
	// Model is the model the measurement is attributed to, "" when unknown.
	Model string
}

BaseMetricsData is embedded by every measurement and carries what they all have: which processor measured it, and against which model.

func (d BaseMetricsData) MetricsModel() string

func (d BaseMetricsData) MetricsProcessor() string

type BaseSwitcherFrame ¶ added in v0.1.0

type BaseSwitcherFrame struct {
	BaseControlFrame
}

BaseSwitcherFrame is embedded by every switcher frame.

func NewBaseSwitcherFrame(typeName string) BaseSwitcherFrame

type BaseSystemFrame ¶

type BaseSystemFrame struct{ BaseFrame }

BaseSystemFrame is embedded by system frames. Construct with NewBaseSystemFrame.

func NewBaseSystemFrame(typeName string) BaseSystemFrame

type BotConnectedFrame ¶ added in v0.1.0

type BotConnectedFrame struct {
	BaseSystemFrame
}

BotConnectedFrame reports that the bot has connected to the transport service. A transport that joins a room of its own pushes it downstream once the join succeeds; a transport that is dialed rather than joining emits nothing. It is a system frame.

func NewBotConnectedFrame() *BotConnectedFrame

type BotSpeakingFrame ¶

type BotSpeakingFrame struct {
	BaseSystemFrame
}

BotSpeakingFrame is emitted periodically while the bot is speaking. It is a system frame.

func NewBotSpeakingFrame() *BotSpeakingFrame

type BotStartedSpeakingFrame ¶

type BotStartedSpeakingFrame struct {
	BaseSystemFrame
}

BotStartedSpeakingFrame indicates the bot started speaking. It is a system frame.

func NewBotStartedSpeakingFrame() *BotStartedSpeakingFrame

type BotStoppedSpeakingFrame ¶

type BotStoppedSpeakingFrame struct {
	BaseSystemFrame
}

BotStoppedSpeakingFrame indicates the bot stopped speaking. It is a system frame.

func NewBotStoppedSpeakingFrame() *BotStoppedSpeakingFrame

type CancelFrame ¶

type CancelFrame struct {
	BaseSystemFrame
	// Reason describes why the pipeline was canceled; "" when unset.
	Reason string
}

CancelFrame indicates the pipeline must stop immediately, without processing any remaining queued frames. It is a system frame.

func NewCancelFrame() *CancelFrame

func (f *CancelFrame) String() string

type CancelWorkerFrame ¶ added in v0.1.0

type CancelWorkerFrame struct {
	BaseSystemFrame
	// Reason describes why the run is being canceled; "" when unset.
	Reason string
}

CancelWorkerFrame requests immediate cancellation of the Task. On reaching it the Task queues a CancelFrame downstream, so the pipeline stops at once without flushing queued frames. It is the deliberate counterpart to a fatal ErrorFrame: use it to stop a run that is not failing (the caller hung up, a supervisor asked to stop).

func NewCancelWorkerFrame() *CancelWorkerFrame

func (f *CancelWorkerFrame) String() string

type ClientConnectedFrame ¶ added in v0.1.0

type ClientConnectedFrame struct {
	BaseSystemFrame
}

ClientConnectedFrame reports that a client has connected to the transport. The input transport pushes it downstream when a participant connects, and an observer measuring how long the transport took to be ready reads it. It is a system frame.

func NewClientConnectedFrame() *ClientConnectedFrame

type ControlFrame ¶

type ControlFrame interface {
	Frame
	// contains filtered or unexported methods
}

ControlFrame is processed in order like a DataFrame and is canceled by user interruptions; it carries control information such as settings updates or a request to end the pipeline once everything is flushed. Embed BaseControlFrame to define one.

type DTMFOutput ¶ added in v0.1.0

type DTMFOutput interface {
	Frame
	// Keys are the keys to emit, in order.
	Keys() []KeypadEntry
}

DTMFOutput is implemented by both output DTMF frames, so a transport can take either without caring which queue it arrived on.

type DataFrame ¶

type DataFrame interface {
	Frame
	// contains filtered or unexported methods
}

DataFrame is processed in order and is canceled by user interruptions. It usually carries data such as LLM context, text, audio or images. Embed BaseDataFrame to define one.

type EndFrame ¶

type EndFrame struct {
	BaseControlFrame
	UninterruptibleMixin
	// Reason describes why the pipeline is ending; "" when unset.
	Reason string
}

EndFrame indicates the pipeline has ended and processors should shut down. As a control frame it is received in order, after preceding frames are flushed. It is uninterruptible so it survives an interruption and the pipeline always shuts down cleanly.

func NewEndFrame() *EndFrame

func (f *EndFrame) String() string

type EndWorkerFrame ¶ added in v0.1.0

type EndWorkerFrame struct {
	BaseControlFrame
	UninterruptibleMixin
	// Reason describes why the run is ending; "" when unset.
	Reason string
}

EndWorkerFrame requests a graceful shutdown of the pipeline worker (the Task). On reaching the Task it queues an EndFrame downstream, so frames already queued are flushed and the bot finishes speaking before the pipeline ends.

func NewEndWorkerFrame() *EndWorkerFrame

func (f *EndWorkerFrame) String() string

type ErrorFrame ¶

type ErrorFrame struct {
	BaseSystemFrame
	// Error describes the error that occurred.
	Error string
	// Fatal reports whether the error is unrecoverable and requires shutdown.
	Fatal bool
	// Source is the processor that raised the error, if known.
	Source ErrorSource
	// Err is the underlying error, if any.
	Err error
	// Category is what kind of failure this was: rejected credentials, an
	// unreachable provider, a malformed request and so on. The zero value means
	// nobody has said yet, which invites the category to be worked out from Err;
	// set it to errors.Unknown to report a failure whose cause cannot be
	// attributed. It is always settled by the time the frame travels.
	Category errors.Category
}

ErrorFrame notifies upstream that an error occurred downstream. A fatal error is unrecoverable and the bot should exit. It is a system frame.

func NewErrorFrame(message string) *ErrorFrame

func (f *ErrorFrame) ErrorInfo() *ErrorFrame

func (f *ErrorFrame) String() string

type ErrorReport ¶ added in v0.1.0

type ErrorReport interface {
	Frame
	// ErrorInfo returns the error the frame carries.
	ErrorInfo() *ErrorFrame
}

ErrorReport is implemented by every frame that reports an error: ErrorFrame and the frames embedding it, such as FatalErrorFrame. Match on it rather than on ErrorFrame itself, which a frame that embeds it does not satisfy, so an error reported by type is not missed.

type ErrorSource ¶

type ErrorSource interface {
	Name() string
	Usable() bool
}

ErrorSource identifies the component that raised an error, in practice the frame processor that produced it. It is declared here, rather than imported from the processor package, so the frames package keeps no dependency on it; a frame processor satisfies this interface by exposing its name and whether it can still do its job.

Usable is settled before the frame travels, so a handler reading it sees the verdict that came with the error it is handling rather than a later one.

type FatalErrorFrame ¶ added in v0.1.0

type FatalErrorFrame struct {
	ErrorFrame
}

FatalErrorFrame notifies upstream that an unrecoverable error occurred and the bot should exit immediately. It is an ErrorFrame whose Fatal is always set, so a processor can report an unrecoverable failure by type rather than by remembering to set the flag.

func NewFatalErrorFrame(message string) *FatalErrorFrame

type FilterControlBase ¶ added in v0.1.0

type FilterControlBase struct{ BaseControlFrame }

FilterControlBase is embedded by filter control frames.

type FilterControlFrame ¶ added in v0.1.0

type FilterControlFrame interface {
	ControlFrame
	// contains filtered or unexported methods
}

FilterControlFrame is the base for the frames that drive an input transport's audio filter at runtime. Assert this interface to test whether a frame is a filter control; embed FilterControlBase to define one. It is a control frame, so a filter change is ordered against the audio around it.

type FilterEnableFrame ¶ added in v0.1.0

type FilterEnableFrame struct {
	FilterControlBase
	// Enable reports whether the filter should be enabled.
	Enable bool
}

FilterEnableFrame turns the filter on or off at runtime, passing incoming audio through untouched without tearing the filter down.

func NewFilterEnableFrame(enable bool) *FilterEnableFrame

func (f *FilterEnableFrame) String() string

type FilterUpdateSettingsFrame ¶ added in v0.1.0

type FilterUpdateSettingsFrame struct {
	FilterControlBase
	// Settings are the filter settings to apply.
	Settings map[string]any
}

FilterUpdateSettingsFrame updates the filter's settings, for example to change how strongly it suppresses noise. The Settings are interpreted by the filter implementation.

func NewFilterUpdateSettingsFrame(settings map[string]any) *FilterUpdateSettingsFrame

func (f *FilterUpdateSettingsFrame) String() string

type Frame ¶

type Frame interface {
	fmt.Stringer

	// ID is a process-unique identifier for this frame instance.
	ID() uint64
	// Name is a human-readable label, "<TypeName>#<n>".
	Name() string

	// Base exposes the embedded BaseFrame and the optional state it carries.
	Base() *BaseFrame
	// contains filtered or unexported methods
}

Frame is implemented by every frame that flows through a pipeline. Concrete frames embed BaseFrame (directly or via BaseSystemFrame, BaseDataFrame or BaseControlFrame), which supplies these methods.

The unexported isFrame marker means a type must embed BaseFrame to satisfy Frame; this guarantees every Frame has a valid id and name. Frames carry mutable state and are passed as pointers.

The interface is deliberately narrow: identity is all a pipeline needs to route, log and correlate a frame. The optional per-frame state — presentation timestamp, metadata, transport source and destination, broadcast sibling id — lives on BaseFrame and is reached through Base. Those accessors are also promoted onto every concrete frame, so a caller holding a concrete type can keep calling them directly.

type FrameProcessorPauseFrame ¶ added in v0.1.0

type FrameProcessorPauseFrame struct {
	BaseControlFrame
	// Processor is the processor to pause.
	Processor ProcessorTarget
}

FrameProcessorPauseFrame asks a processor to pause its handling of data and control frames. Paused frames stay in the processor's queue and are handled once processing resumes with a FrameProcessorResumeFrame. It is a control frame, so it is received in order, after the frames queued ahead of it; use FrameProcessorPauseUrgentFrame to pause immediately instead.

func NewFrameProcessorPauseFrame(p ProcessorTarget) *FrameProcessorPauseFrame

type FrameProcessorPauseUrgentFrame ¶ added in v0.1.0

type FrameProcessorPauseUrgentFrame struct {
	BaseSystemFrame
	// Processor is the processor to pause.
	Processor ProcessorTarget
}

FrameProcessorPauseUrgentFrame asks a processor to pause its handling of data and control frames as fast as possible. Paused frames stay in the processor's queue and are handled once processing resumes. It is a system frame, so it overtakes the frames queued ahead of it; use FrameProcessorPauseFrame to pause in order instead.

func NewFrameProcessorPauseUrgentFrame(p ProcessorTarget) *FrameProcessorPauseUrgentFrame

type FrameProcessorResumeFrame ¶ added in v0.1.0

type FrameProcessorResumeFrame struct {
	BaseControlFrame
	// Processor is the processor to resume.
	Processor ProcessorTarget
}

FrameProcessorResumeFrame asks a processor to resume the handling of data and control frames it paused. Queued frames are then handled in the order they were received. It is a control frame, so it is received in order, after the frames queued ahead of it; use FrameProcessorResumeUrgentFrame to resume immediately instead.

func NewFrameProcessorResumeFrame(p ProcessorTarget) *FrameProcessorResumeFrame

type FrameProcessorResumeUrgentFrame ¶ added in v0.1.0

type FrameProcessorResumeUrgentFrame struct {
	BaseSystemFrame
	// Processor is the processor to resume.
	Processor ProcessorTarget
}

FrameProcessorResumeUrgentFrame asks a processor to resume the handling of data and control frames it paused, as fast as possible. Queued frames are then handled in the order they were received. It is a system frame, so it overtakes the frames queued ahead of it; use FrameProcessorResumeFrame to resume in order instead.

func NewFrameProcessorResumeUrgentFrame(p ProcessorTarget) *FrameProcessorResumeUrgentFrame

type FunctionCallCancelFrame ¶

type FunctionCallCancelFrame struct {
	BaseSystemFrame
	// ToolCallID identifies the canceled call.
	ToolCallID string
	// ToolName is the tool's name.
	ToolName string
	// RunLLM asks for inference once the cancellation is settled in the
	// conversation. Only a call canceled by its own deadline sets it: an
	// interruption must not trigger inference, and a cancellation the model
	// asked for already runs inference through the result of the tool that
	// requested it.
	RunLLM bool
}

FunctionCallCancelFrame reports that a tool call was canceled, which happens when a barge-in interrupts a call registered to be canceled on interruption. The assistant context aggregator marks the call's placeholder result canceled so the pair stays balanced. It is a system frame because cancellation must overtake the queued frames of the turn being abandoned.

func NewFunctionCallCancelFrame(toolCallID, name string) *FunctionCallCancelFrame

func (f *FunctionCallCancelFrame) String() string

type FunctionCallInProgressFrame ¶

type FunctionCallInProgressFrame struct {
	BaseControlFrame
	UninterruptibleMixin
	// ToolCallID is the id of the call that is executing.
	ToolCallID string
	// ToolName is the tool's name.
	ToolName string
	// Args is the raw JSON arguments the model produced for the call.
	Args json.RawMessage
	// CancelOnInterruption reports whether the call is canceled when the turn is
	// interrupted. A call registered with it false is asynchronous: the model
	// carries on without waiting, and the result arrives later as a developer
	// message (see AsyncToolMessage).
	CancelOnInterruption bool
	// GroupID is shared by every call the model requested in one response, so the
	// aggregator can tell when the last of them completes.
	GroupID string
}

FunctionCallInProgressFrame reports that a specific tool call has started executing. The assistant context aggregator writes the assistant tool-call message and a placeholder tool result from it, which is what keeps the pair balanced for every inference that follows. It is uninterruptible because the context must be updated even when the turn is cut off.

func NewFunctionCallInProgressFrame(
	toolCallID, name string, args json.RawMessage, cancelOnInterruption bool, groupID string,
) *FunctionCallInProgressFrame

func (f *FunctionCallInProgressFrame) String() string

type FunctionCallResultFrame ¶

type FunctionCallResultFrame struct {
	BaseDataFrame
	UninterruptibleMixin
	// ToolCallID pairs the result to its call.
	ToolCallID string
	// ToolName is the tool's name.
	ToolName string
	// Args is the raw JSON arguments the call was made with.
	Args json.RawMessage
	// Result is the tool-result content.
	Result string
	// RunLLM forces whether generation re-runs after this result. Nil leaves the
	// decision to the aggregator. Properties, when it sets RunLLM, wins over it.
	RunLLM *bool
	// Properties tunes how the result is applied; nil means the defaults.
	Properties *FunctionCallResultProperties
}

FunctionCallResultFrame carries the result of one tool call. The assistant context aggregator updates the call's placeholder result in place, so the tool-use and tool-result pair stays adjacent and balanced. It is uninterruptible because a result that was produced must always reach the context.

func NewFunctionCallResultFrame(toolCallID, name string, args json.RawMessage, result string) *FunctionCallResultFrame

func (f *FunctionCallResultFrame) String() string

type FunctionCallResultProperties ¶ added in v0.1.0

type FunctionCallResultProperties struct {
	// RunLLM forces whether generation re-runs after this result. Nil leaves the
	// decision to the aggregator, which re-runs once the last call in the group
	// completes.
	RunLLM *bool
	// OnContextUpdated runs once the result has been written to the context. It
	// runs on its own goroutine so it never blocks the pipeline.
	OnContextUpdated func(ctx context.Context) error
	// IsFinal reports whether this is the call's final result. Nil means final.
	// A false value marks an intermediate update, which is only meaningful for an
	// asynchronous call (one registered with CancelOnInterruption false).
	IsFinal *bool
}

FunctionCallResultProperties tunes how a tool result is applied. A handler passes it to its result callback; a nil value means the defaults, which are to treat the result as final and to let the aggregator decide whether to re-generate.

func (p *FunctionCallResultProperties) Final() bool

type FunctionCallsStartedFrame ¶

type FunctionCallsStartedFrame struct {
	BaseSystemFrame
	// Calls are the tool calls the model requested this turn.
	Calls []ToolCall
}

FunctionCallsStartedFrame announces that the model requested one or more tool calls in the current assistant turn. The assistant context aggregator records the calls as awaiting results; it writes nothing to the context, because each call's own FunctionCallInProgressFrame writes the assistant tool-call message and its placeholder result. It is a system frame so it reaches the aggregator ahead of the queued frames of the turn it belongs to.

func NewFunctionCallsStartedFrame(calls []ToolCall) *FunctionCallsStartedFrame

func (f *FunctionCallsStartedFrame) String() string

type HeartbeatFrame ¶ added in v0.1.0

type HeartbeatFrame struct {
	BaseControlFrame
	// Timestamp is the pipeline clock reading when the heartbeat was created.
	// Comparing it with the clock on arrival gives the time the frame spent
	// crossing the pipeline.
	Timestamp time.Duration
}

HeartbeatFrame is pushed through the pipeline at a fixed interval so the Task can tell frames are still moving. It travels in order like any control frame, which is what makes it a measure of the path real work takes. It is interruptible on purpose: a barge-in drops the ones in flight, and the next interval sends another. It is a control frame.

func NewHeartbeatFrame(ts time.Duration) *HeartbeatFrame

func (f *HeartbeatFrame) String() string

type InputAudioRawFrame ¶

type InputAudioRawFrame struct {
	BaseSystemFrame
	AudioRawData
}

InputAudioRawFrame is a chunk of audio coming from an input transport. When a transport exposes multiple audio sources, the source name is carried in TransportSource. It is a system frame.

func NewInputAudioRawFrame(audio []byte, sampleRate, numChannels int) *InputAudioRawFrame

func (f *InputAudioRawFrame) String() string

type InputDTMFFrame ¶ added in v0.0.4

type InputDTMFFrame struct {
	BaseSystemFrame
	// Button is the key that was pressed.
	Button KeypadEntry
}

InputDTMFFrame is a DTMF keypress received from the transport — for example a phone caller pressing a key. It is a system frame so it is delivered with priority and in order.

func NewInputDTMFFrame(button KeypadEntry) *InputDTMFFrame

func (f *InputDTMFFrame) String() string

type InputTextRawFrame ¶ added in v0.1.0

type InputTextRawFrame struct {
	BaseSystemFrame
	// Text is the text that arrived.
	Text string
}

InputTextRawFrame is text arriving from a transport as input, usually because the user typed it or an application injected it, and meant for the LLM the same way spoken input is. It is the text counterpart of InputAudioRawFrame, and a system frame so it reaches the pipeline ahead of the queued conversation.

func NewInputTextRawFrame(text string) *InputTextRawFrame

func (f *InputTextRawFrame) String() string

type InputTransportMessageFrame ¶

type InputTransportMessageFrame struct {
	BaseSystemFrame
	// Message is the raw message payload as received (typically JSON).
	Message []byte
}

InputTransportMessageFrame carries an application message received by a transport from the client — for example an RTVI message off a WebRTC data channel. It is a system frame so it is handled with priority and in order.

func NewInputTransportMessageFrame(message []byte) *InputTransportMessageFrame

func (f *InputTransportMessageFrame) String() string

type InputTransportStartAudioStreamingFrame ¶ added in v0.1.0

type InputTransportStartAudioStreamingFrame struct {
	BaseControlFrame
}

InputTransportStartAudioStreamingFrame asks the input transport to start streaming audio from its source. It is pushed downstream (by the RTVI processor once the client is ready, say) so that starting the stream stays frame-based rather than a direct call across processors. It is a control frame.

func NewInputTransportStartAudioStreamingFrame() *InputTransportStartAudioStreamingFrame

type InterimTranscriptionFrame ¶

type InterimTranscriptionFrame struct {
	TextFrame
	// UserID identifies the user who spoke.
	UserID string
	// Timestamp is when the interim transcription occurred.
	Timestamp string
	// Language is the detected or specified language as a BCP-47 tag; "" when
	// unset.
	Language string
	// Result is the raw result from the STT service, if available.
	Result any
}

InterimTranscriptionFrame carries a partial (non-final) speech transcription for a user.

func NewInterimTranscriptionFrame(text, userID, timestamp string) *InterimTranscriptionFrame

func (f *InterimTranscriptionFrame) String() string

type InterruptionFrame ¶

type InterruptionFrame struct {
	BaseSystemFrame
}

InterruptionFrame interrupts the pipeline — for example when the user starts speaking, to cancel in-progress bot output. It can be pushed by any processor. It is a system frame.

func NewInterruptionFrame() *InterruptionFrame

type InterruptionWorkerFrame ¶ added in v0.1.0

type InterruptionWorkerFrame struct {
	BaseSystemFrame
}

InterruptionWorkerFrame asks the Task to interrupt the pipeline. On reaching the Task it queues an InterruptionFrame downstream. A processor that can broadcast an InterruptionFrame itself should do so; this frame is for one that only has a path to the Task.

func NewInterruptionWorkerFrame() *InterruptionWorkerFrame

type KeypadEntry ¶ added in v0.0.4

type KeypadEntry string

KeypadEntry is a single DTMF keypad key: the digits 0-9, the symbols * and #, and the letters A-D.

const (
	KeypadZero  KeypadEntry = "0"
	KeypadOne   KeypadEntry = "1"
	KeypadTwo   KeypadEntry = "2"
	KeypadThree KeypadEntry = "3"
	KeypadFour  KeypadEntry = "4"
	KeypadFive  KeypadEntry = "5"
	KeypadSix   KeypadEntry = "6"
	KeypadSeven KeypadEntry = "7"
	KeypadEight KeypadEntry = "8"
	KeypadNine  KeypadEntry = "9"
	KeypadStar  KeypadEntry = "*"
	KeypadPound KeypadEntry = "#"
	KeypadA     KeypadEntry = "A"
	KeypadB     KeypadEntry = "B"
	KeypadC     KeypadEntry = "C"
	KeypadD     KeypadEntry = "D"
)

func (e KeypadEntry) Valid() bool

type LLMAssistantPushAggregationFrame ¶ added in v0.1.0

type LLMAssistantPushAggregationFrame struct {
	BaseControlFrame
}

LLMAssistantPushAggregationFrame makes the assistant aggregator commit what it has gathered to the conversation as an assistant message, without waiting for an LLMFullResponseEndFrame. It closes out an utterance the service spoke on its own, which has no LLM response around it to end. It is a control frame.

func NewLLMAssistantPushAggregationFrame() *LLMAssistantPushAggregationFrame

type LLMConfigureOutputFrame ¶ added in v0.1.0

type LLMConfigureOutputFrame struct {
	BaseDataFrame
	// SkipTTS reports whether the tokens the LLM emits should skip the TTS
	// service, if the pipeline has one.
	SkipTTS bool
}

LLMConfigureOutputFrame configures how an LLM service produces output. It tells the service to stamp the tokens it emits so a TTS service downstream passes them through instead of speaking them: the reply is added to the conversation but never said out loud. It is a data frame.

func NewLLMConfigureOutputFrame(skipTTS bool) *LLMConfigureOutputFrame

type LLMContext ¶

type LLMContext struct {
	// contains filtered or unexported fields
}

LLMContext holds the conversation so far: a system prompt plus the running list of user and assistant messages. The user and assistant aggregators append to a shared context as the conversation proceeds, and the LLM service reads it to generate each response. It is safe for concurrent use.

func NewLLMContext(system string) *LLMContext

func (c *LLMContext) AddAssistantMessage(text string)

func (c *LLMContext) AddAssistantToolCall(call ToolCall)

func (c *LLMContext) AddMessage(m Message)

func (c *LLMContext) AddToolResult(r ToolResult)

func (c *LLMContext) AddUserMessage(text string)

func (c *LLMContext) Messages() []Message

func (c *LLMContext) MessagesFor(llm string) []Message

func (c *LLMContext) Recall() string

func (c *LLMContext) ReplaceLastAssistantText(text string) bool

func (c *LLMContext) SetMessages(messages []Message)

func (c *LLMContext) SetRecall(recall string)

func (c *LLMContext) SetSystem(system string)

func (c *LLMContext) SetToolChoice(choice ToolChoice)

func (c *LLMContext) SetTools(tools []Tool)

func (c *LLMContext) SetToolsSchema(schema ToolsSchema)

func (c *LLMContext) System() string

func (c *LLMContext) SystemParts() (stable, volatile string)

func (c *LLMContext) ToolChoice() ToolChoice

func (c *LLMContext) Tools() []Tool

func (c *LLMContext) ToolsSchema() ToolsSchema

func (c *LLMContext) TransformMessages(transform func([]Message) []Message)

func (c *LLMContext) UpdateToolResult(toolCallID, content string) bool

type LLMContextAssistantTimestampFrame ¶ added in v0.1.0

type LLMContextAssistantTimestampFrame struct {
	BaseDataFrame
	// Timestamp is when the assistant message was created, in ISO 8601.
	Timestamp string
}

LLMContextAssistantTimestampFrame carries when an assistant message was written, for a consumer keeping its own record of the conversation alongside the context. It is a data frame.

func NewLLMContextAssistantTimestampFrame(timestamp string) *LLMContextAssistantTimestampFrame

func (f *LLMContextAssistantTimestampFrame) String() string

type LLMContextAssistantTurnFrame ¶ added in v0.1.0

type LLMContextAssistantTurnFrame struct {
	BaseDataFrame
	// Text is what the assistant said this turn.
	Text string
	// Timestamp is when the assistant's turn began, in ISO 8601.
	Timestamp string
}

LLMContextAssistantTurnFrame carries the aggregated text of a completed assistant turn.

The assistant aggregator broadcasts it as a turn ends, carrying the same text it wrote to the conversation, so processors on either side can act on each completed reply without an observer of their own. It is a data frame.

func NewLLMContextAssistantTurnFrame(text, timestamp string) *LLMContextAssistantTurnFrame

func (f *LLMContextAssistantTurnFrame) String() string

type LLMContextFrame ¶

type LLMContextFrame struct {
	BaseDataFrame
	// Context is the conversation to generate a response from.
	Context *LLMContext
}

LLMContextFrame carries the conversation context to the LLM service to trigger a response. It is a data frame.

func NewLLMContextFrame(ctx *LLMContext) *LLMContextFrame

func (f *LLMContextFrame) String() string

type LLMContextSummaryRequestFrame ¶ added in v0.1.0

type LLMContextSummaryRequestFrame struct {
	BaseControlFrame

	// RequestID matches this request to the result answering it, so a result
	// arriving for a request that has since been abandoned is recognized and
	// dropped.
	RequestID string
	// Context is the conversation to summarize.
	Context *LLMContext
	// MinMessagesToKeep is how many of the most recent messages stay out of the
	// summary.
	MinMessagesToKeep int
	// TargetContextTokens caps the length of the generated summary.
	TargetContextTokens int
	// SummarizationPrompt tells the model how to summarize.
	SummarizationPrompt string
	// SummarizationTimeout bounds this generation. Zero uses
	// DefaultSummarizationTimeout.
	SummarizationTimeout time.Duration
}

LLMContextSummaryRequestFrame asks the LLM service to generate a summary of the conversation. The summarizer pushes it upstream; the service answers with an LLMContextSummaryResultFrame carrying the same RequestID.

func NewLLMContextSummaryRequestFrame(requestID string, convo *LLMContext) *LLMContextSummaryRequestFrame

func (f *LLMContextSummaryRequestFrame) String() string

type LLMContextSummaryResultFrame ¶ added in v0.1.0

type LLMContextSummaryResultFrame struct {
	BaseControlFrame
	UninterruptibleMixin

	// RequestID identifies the request this answers.
	RequestID string
	// Summary is the generated summary, empty when Error is set.
	Summary string
	// LastSummarizedIndex is the index of the last message the summary covers,
	// and -1 when nothing was summarized.
	LastSummarizedIndex int
	// Error describes a summarization that failed, and is empty on success.
	Error string
}

LLMContextSummaryResultFrame carries a generated summary back to the summarizer that asked for it.

It is uninterruptible: the request that produced it may well have been canceled by a barge-in, and the summarizer still has to see the result so it can clear the request it is holding open. Dropping it would leave summarization blocked for the rest of the call.

func NewLLMContextSummaryErrorFrame(requestID, message string) *LLMContextSummaryResultFrame

func NewLLMContextSummaryResultFrame(requestID, summary string, lastSummarizedIndex int) *LLMContextSummaryResultFrame

func (f *LLMContextSummaryResultFrame) String() string

type LLMEnablePromptCachingFrame ¶ added in v0.1.0

type LLMEnablePromptCachingFrame struct {
	BaseDataFrame
	// Enable reports whether prompt caching should be on.
	Enable bool
}

LLMEnablePromptCachingFrame turns a provider's prompt caching on or off, so the conversation's stable prefix is cached and read back on the next turn rather than being charged for again. It is a data frame, so it is ordered against the conversation it applies to.

func NewLLMEnablePromptCachingFrame(enable bool) *LLMEnablePromptCachingFrame

func (f *LLMEnablePromptCachingFrame) String() string

type LLMFullResponseEndFrame ¶

type LLMFullResponseEndFrame struct {
	BaseControlFrame
	// SkipTTS, when set, reports whether the response should skip TTS. A nil
	// value means "unset".
	SkipTTS *bool
}

LLMFullResponseEndFrame marks the end of an LLM response. It is a control frame.

func NewLLMFullResponseEndFrame() *LLMFullResponseEndFrame

type LLMFullResponseStartFrame ¶

type LLMFullResponseStartFrame struct {
	BaseControlFrame
	// SkipTTS, when set, reports whether the response should skip TTS. A nil
	// value means "unset".
	SkipTTS *bool
}

LLMFullResponseStartFrame marks the beginning of an LLM response, followed by one or more TextFrames and a final LLMFullResponseEndFrame. It is a control frame.

func NewLLMFullResponseStartFrame() *LLMFullResponseStartFrame

type LLMMarkerFrame ¶

type LLMMarkerFrame struct {
	BaseDataFrame
	// Marker is the marker text.
	Marker string
	// AppendToContextImmediately asks for the marker to be written to the
	// conversation as an assistant message of its own, as soon as it arrives.
	// Clear it to have the marker held and flushed together with the text that
	// follows, as one message, which is what makes a gated reply read as the
	// marker followed by the response rather than as two entries.
	AppendToContextImmediately bool
}

LLMMarkerFrame carries a turn-completion marker the LLM emitted (for example "●"). It is informational, since the TTS downstream ignores it, and lets observers see the model's completeness verdict. It is a data frame.

func NewLLMMarkerFrame(marker string) *LLMMarkerFrame

type LLMMessagesAppendFrame ¶

type LLMMessagesAppendFrame struct {
	BaseDataFrame
	// Messages are the messages to append.
	Messages []Message
	// RunLLM reports whether the LLM should run on the updated context.
	RunLLM bool
}

LLMMessagesAppendFrame asks the context aggregator to append messages to the LLM context, in contrast to LLMMessagesUpdateFrame which replaces them. The turn-completion re-prompt and a conversation flow entering a node both use it. It is a data frame, so the messages are ordered against the surrounding conversation.

func NewLLMMessagesAppendFrame(messages []Message) *LLMMessagesAppendFrame

type LLMMessagesTransformFrame ¶ added in v0.1.0

type LLMMessagesTransformFrame struct {
	BaseDataFrame
	// Transform takes the context's current messages and returns what they
	// should become.
	Transform func([]Message) []Message
	// RunLLM reports whether the LLM should run on the rewritten context.
	RunLLM bool
}

LLMMessagesTransformFrame rewrites the conversation messages in the shared context with a function of them, in contrast to LLMMessagesUpdateFrame which replaces them with a list settled in advance. Use it when what the conversation should become depends on what it currently is: redacting what a tool returned, dropping the turns about a topic the user asked to forget. It is a data frame, so the rewrite is ordered against the surrounding conversation.

func NewLLMMessagesTransformFrame(transform func([]Message) []Message) *LLMMessagesTransformFrame

func (f *LLMMessagesTransformFrame) String() string

type LLMMessagesUpdateFrame ¶ added in v0.1.0

type LLMMessagesUpdateFrame struct {
	BaseDataFrame
	// Messages replaces the context's current messages.
	Messages []Message
	// RunLLM reports whether the LLM should run on the updated context.
	RunLLM bool
}

LLMMessagesUpdateFrame replaces the conversation messages in the shared context, in contrast to LLMMessagesAppendFrame which adds to them. Use it to swap the conversation wholesale — restoring a saved session, or resetting the conversation without rebuilding the pipeline. It is a data frame, so the replacement is ordered against the surrounding conversation.

func NewLLMMessagesUpdateFrame(messages []Message) *LLMMessagesUpdateFrame

func (f *LLMMessagesUpdateFrame) String() string

type LLMRunFrame ¶

type LLMRunFrame struct {
	BaseDataFrame
}

LLMRunFrame instructs the LLM service to process the current context and generate a response. Queue it to make the bot speak first at the start of a session, or to re-run after editing the context. It carries no data — the user aggregator runs its current shared context. It is a data frame.

func NewLLMRunFrame() *LLMRunFrame

type LLMServiceMetadataFrame ¶ added in v0.0.4

type LLMServiceMetadataFrame struct {
	ServiceMetadataFrame
	// Realtime reports whether the broadcasting LLM is a realtime
	// (speech-to-speech) service.
	Realtime bool
}

LLMServiceMetadataFrame is the metadata an LLM service broadcasts. It reports whether the service is a realtime (speech-to-speech) LLM.

func NewLLMServiceMetadataFrame(service string) *LLMServiceMetadataFrame

type LLMSetToolChoiceFrame ¶ added in v0.1.0

type LLMSetToolChoiceFrame struct {
	BaseDataFrame
	// ToolChoice is the new setting.
	ToolChoice ToolChoice
}

LLMSetToolChoiceFrame changes whether the model may or must call a tool. Like LLMSetToolsFrame it is applied to the shared context and forwarded downstream so realtime services learn of the change. It is a data frame.

func NewLLMSetToolChoiceFrame(choice ToolChoice) *LLMSetToolChoiceFrame

func (f *LLMSetToolChoiceFrame) String() string

type LLMSetToolsFrame ¶ added in v0.1.0

type LLMSetToolsFrame struct {
	BaseDataFrame
	// Tools is the new toolset; nil or empty clears the tools.
	Tools []Tool
}

LLMSetToolsFrame changes the set of tools advertised to the model mid-conversation. The context aggregator applies it to the shared context and forwards it downstream: a text LLM picks the change up on its next run, but a realtime (speech-to-speech) service is generating continuously and would never see it, so it must be told. Always change tools through this frame rather than calling LLMContext.SetTools directly, or realtime services will keep using the old toolset. It is a data frame, so the change is ordered against the surrounding conversation instead of racing an in-flight generation.

func NewLLMSetToolsFrame(tools []Tool) *LLMSetToolsFrame

func (f *LLMSetToolsFrame) String() string

type LLMSummarizeContextFrame ¶ added in v0.1.0

type LLMSummarizeContextFrame struct {
	BaseControlFrame

	// Config overrides the summary generation settings for this request alone.
	// Nil uses the configured defaults.
	Config *SummaryConfig
}

LLMSummarizeContextFrame asks for the conversation to be compressed now, whatever the automatic thresholds say. Push it into the pipeline to compress on demand; it works whether or not automatic summarization is enabled.

It is a control frame, so it is handled in order with the conversation around it rather than jumping ahead of a turn in flight.

func NewLLMSummarizeContextFrame() *LLMSummarizeContextFrame

type LLMTextFrame ¶

type LLMTextFrame struct {
	TextFrame
}

LLMTextFrame is a TextFrame produced by an LLM service. LLM output already includes any necessary inter-frame spaces.

func NewLLMTextFrame(text string) *LLMTextFrame

type LLMThoughtEndFrame ¶ added in v0.1.0

type LLMThoughtEndFrame struct {
	BaseControlFrame
	// Signature is what the provider signs the thought with, where it signs one.
	// It is carried back unread, since only that provider can make sense of it.
	Signature any
}

LLMThoughtEndFrame marks the end of a thought. It is a control frame.

func NewLLMThoughtEndFrame() *LLMThoughtEndFrame

func (f *LLMThoughtEndFrame) String() string

type LLMThoughtStartFrame ¶ added in v0.1.0

type LLMThoughtStartFrame struct {
	BaseControlFrame
	// AppendToContext reports whether the thought should be written to the
	// conversation. A thought that is written goes in as a message of the
	// provider's own, so LLM must name that provider when this is set.
	AppendToContext bool
	// LLM identifies the provider whose native message the thought is written
	// as. It is only read when AppendToContext is set.
	LLM string
}

LLMThoughtStartFrame marks the beginning of a thought from a reasoning model, followed by one or more LLMThoughtTextFrames and a final LLMThoughtEndFrame. It is a control frame.

func NewLLMThoughtStartFrame() *LLMThoughtStartFrame

func (f *LLMThoughtStartFrame) String() string

type LLMThoughtTextFrame ¶ added in v0.1.0

type LLMThoughtTextFrame struct {
	BaseDataFrame
	// Text is the thought, or a chunk of it.
	Text string
	// IncludesInterFrameSpaces reports whether the spacing between chunks is
	// already part of Text. A thought's chunks always carry their own spacing.
	IncludesInterFrameSpaces bool
}

LLMThoughtTextFrame carries the text of a thought, or a chunk of one.

Despite carrying text it is a data frame rather than a TextFrame, which is what keeps it out of the ordinary text handling: a thought is the model reasoning with itself and must not be spoken.

func NewLLMThoughtTextFrame(text string) *LLMThoughtTextFrame

func (f *LLMThoughtTextFrame) String() string

type LLMTokenUsage ¶

type LLMTokenUsage struct {
	// PromptTokens is the number of input tokens. It is net of the cache on a
	// service that reports its cache reads separately.
	PromptTokens int64
	// CompletionTokens is the number of output tokens.
	CompletionTokens int64
	// TotalTokens is every token the generation used, cached input included.
	TotalTokens int64
	// CacheReadTokens is the number of input tokens read from the prompt cache.
	CacheReadTokens *int64
	// CacheCreationTokens is the number of input tokens written to the prompt cache.
	CacheCreationTokens *int64
	// ReasoningTokens is the number of completion tokens the model spent
	// reasoning before answering, on a model that reports them. It is a subset
	// of CompletionTokens.
	ReasoningTokens *int64
	// InputAudioTokens is the number of input (prompt) tokens that were audio,
	// as reported by realtime models. It is a subset of PromptTokens.
	InputAudioTokens *int64
	// OutputAudioTokens is the number of output (completion) tokens that were
	// audio. It is a subset of CompletionTokens.
	OutputAudioTokens *int64
	// CacheReadAudioTokens is the number of cache-read input tokens that were
	// audio. It is a subset of CacheReadTokens, and realtime models price it
	// apart from cached text.
	CacheReadAudioTokens *int64
	// InputTextTokens is the number of input (prompt) tokens that were text,
	// when the model reports a per-modality breakdown. Subset of PromptTokens.
	InputTextTokens *int64
	// OutputTextTokens is the number of output (completion) tokens that were
	// text, when the model reports a per-modality breakdown. Subset of
	// CompletionTokens.
	OutputTextTokens *int64
}

LLMTokenUsage reports the token counts billed for one LLM generation. CacheReadTokens were served from a prompt cache and CacheCreationTokens were written to one.

Services differ over whether they report the input count net or gross of the cache. TotalTokens is the gross figure either way, so it stays comparable between services, and it is therefore not always PromptTokens plus CompletionTokens. Read the cache counts for the breakdown rather than subtracting.

The per-modality audio and text counts are subsets, of the prompt tokens (input) and completion tokens (output). Realtime (speech-to-speech) models bill audio and text at different rates and report this breakdown.

The counts a service may or may not account for are pointers, so that a service which reports a figure of zero is distinguishable from one that does not report the figure at all. A model with no prompt cache leaves CacheReadTokens nil; a model with one that served nothing from it this generation reports zero. Only the second is worth showing on a cost dashboard, and only the pointer tells them apart.

type LLMUpdateSettingsFrame ¶ added in v0.1.0

type LLMUpdateSettingsFrame struct {
	ServiceUpdateSettingsFrame
}

LLMUpdateSettingsFrame changes a language model service's settings.

func NewLLMUpdateSettingsFrame(delta any) *LLMUpdateSettingsFrame

func (f *LLMUpdateSettingsFrame) Copy() SettingsUpdate

func (f *LLMUpdateSettingsFrame) String() string

type LLMUsageMetricsData ¶ added in v0.1.0

type LLMUsageMetricsData struct {
	BaseMetricsData
	// Value is the token usage.
	Value LLMTokenUsage
}

LLMUsageMetricsData is the token usage billed for one LLM generation.

type ManuallySwitchServiceFrame ¶ added in v0.1.0

type ManuallySwitchServiceFrame struct {
	BaseSwitcherFrame
	// Service is the service to activate.
	Service ServiceTarget
}

ManuallySwitchServiceFrame asks a service switcher to make Service the active one. A switcher that does not manage Service leaves the frame alone, so the request reaches the switcher that does. It is a control frame.

func NewManuallySwitchServiceFrame(svc ServiceTarget) *ManuallySwitchServiceFrame

func (f *ManuallySwitchServiceFrame) String() string

type Message ¶

type Message struct {
	Role Role
	Text string
	// ToolCalls is set on an assistant message that requested tool calls.
	ToolCalls []ToolCall
	// ToolResults is set on a message returning the outputs of tool calls.
	ToolResults []ToolResult
	// LLM names the provider this message is written for. When it is set the
	// message is that provider's own and no other's: only its adapter sends it,
	// and every other adapter leaves it out (see LLMContext.MessagesFor).
	LLM string
	// Native is the message as that provider's API takes it, in the type that
	// provider's adapter defines. It is only read when LLM is set, and the
	// adapter it names is the only thing that knows how to read it.
	Native any
}

Message is a single conversation turn. A plain turn carries Text; an assistant turn that invoked tools also carries ToolCalls; a turn returning tool outputs carries ToolResults.

A message may instead be written in one provider's own format, which is what LLM and Native carry. See NewLLMSpecificMessage.

func NewAsyncToolCanceledMessage(toolCallID string) Message

func NewAsyncToolFinalMessage(toolCallID, result string) Message

func NewAsyncToolIntermediateMessage(toolCallID, result string) Message

func NewAsyncToolStartedMessage(toolCallID string) Message

func NewLLMSpecificMessage(llm string, native any) Message

func (m Message) IsLLMSpecific() bool

type MetricsData ¶ added in v0.1.0

type MetricsData interface {
	// MetricsProcessor is the name of the processor that measured it.
	MetricsProcessor() string
	// MetricsModel is the model it is attributed to, "" when unknown.
	MetricsModel() string
	// contains filtered or unexported methods
}

MetricsData is one measurement carried by a MetricsFrame. The concrete types are the kinds a processor can report: TTFBMetricsData, TTFAMetricsData, ProcessingMetricsData, LLMUsageMetricsData, STTUsageMetricsData, TTSUsageMetricsData, TextAggregationMetricsData and TurnMetricsData. A consumer switches on the type to read the value.

type MetricsFrame ¶

type MetricsFrame struct {
	BaseSystemFrame
	// Data is the measurements this frame reports. One frame can carry several
	// kinds, and measurements from more than one processor.
	Data []MetricsData
}

MetricsFrame reports measurements made by one or more processors. It is a system frame, so it is delivered with priority and is not dropped by an interruption: usage is billed even when a turn is cut short.

func NewMetricsFrame(data ...MetricsData) *MetricsFrame

func (f *MetricsFrame) String() string

type MixerControlBase ¶ added in v0.1.0

type MixerControlBase struct{ BaseControlFrame }

MixerControlBase is embedded by mixer control frames.

type MixerControlFrame ¶ added in v0.0.4

type MixerControlFrame interface {
	ControlFrame
	// contains filtered or unexported methods
}

MixerControlFrame is the base for the frames that drive an output transport's audio mixer at runtime. Assert this interface to test whether a frame is a mixer control; embed MixerControlBase to define one. It is a control frame, so a mixer change is ordered against the audio around it.

type MixerEnableFrame ¶ added in v0.1.0

type MixerEnableFrame struct {
	MixerControlBase
	// Enable reports whether the mixer should be enabled.
	Enable bool
}

MixerEnableFrame turns the mixer on or off at runtime, muting or restoring the auxiliary audio without changing its settings.

func NewMixerEnableFrame(enable bool) *MixerEnableFrame

func (f *MixerEnableFrame) String() string

type MixerUpdateSettingsFrame ¶ added in v0.1.0

type MixerUpdateSettingsFrame struct {
	MixerControlBase
	// Settings are the mixer settings to apply (e.g. "volume", "track").
	Settings map[string]any
}

MixerUpdateSettingsFrame updates the mixer's settings — for example to change the background track or adjust its volume. The Settings are interpreted by the mixer implementation.

func NewMixerUpdateSettingsFrame(settings map[string]any) *MixerUpdateSettingsFrame

func (f *MixerUpdateSettingsFrame) String() string

type OutputAudioFrame ¶ added in v0.1.0

type OutputAudioFrame interface {
	AudioFrame
	// contains filtered or unexported methods
}

OutputAudioFrame is implemented by every frame carrying audio bound for an output transport: plain output audio, TTS audio, and speech-stream audio. A transport takes one rather than a bare buffer, so it can read the destination the frame names alongside the samples and send it on the right outgoing stream. Assert this interface rather than a concrete type, so audio keeps being handled whichever kind produced it.

type OutputAudioRawFrame ¶

type OutputAudioRawFrame struct {
	BaseDataFrame
	AudioRawData
}

OutputAudioRawFrame is a chunk of audio to be played by an output transport. When a transport exposes multiple audio destinations, the destination name is carried in TransportDestination. It is a data frame.

func NewOutputAudioRawFrame(audio []byte, sampleRate, numChannels int) *OutputAudioRawFrame

func (f *OutputAudioRawFrame) String() string

type OutputDTMFFrame ¶ added in v0.0.4

type OutputDTMFFrame struct {
	BaseControlFrame
	// Buttons are the keys to emit, in order.
	Buttons []KeypadEntry
}

OutputDTMFFrame requests the transport play the DTMF tones for Buttons, to navigate an IVR menu for example. It is a control frame, delivered in order behind the audio already queued, so the keys land where the caller meant them rather than over whatever is still being said.

func NewOutputDTMFFrame(button KeypadEntry) *OutputDTMFFrame

func NewOutputDTMFSequenceFrame(buttons []KeypadEntry) *OutputDTMFFrame

func (f *OutputDTMFFrame) Keys() []KeypadEntry

func (f *OutputDTMFFrame) String() string

type OutputDTMFUrgentFrame ¶ added in v0.1.0

type OutputDTMFUrgentFrame struct {
	BaseSystemFrame
	// Buttons are the keys to emit, in order.
	Buttons []KeypadEntry
}

OutputDTMFUrgentFrame requests the same tones as OutputDTMFFrame but sends them at once, ahead of the audio already queued. It is a system frame, for a keypress that answers a prompt still playing.

func NewOutputDTMFUrgentFrame(button KeypadEntry) *OutputDTMFUrgentFrame

func (f *OutputDTMFUrgentFrame) Keys() []KeypadEntry

func (f *OutputDTMFUrgentFrame) String() string

type OutputTransportMessageFrame ¶

type OutputTransportMessageFrame struct {
	BaseDataFrame
	// Message is the message payload to send; the transport serializes it.
	Message any
}

OutputTransportMessageFrame carries an application message to send to the client over the transport — for example an RTVI message onto a WebRTC data channel. Message is serialized by the output transport. It is a data frame, so it is delivered in order with the surrounding audio: use it for a message that must land in step with what the bot is saying. For a message that must go out immediately, ahead of any queued audio, use OutputTransportMessageUrgentFrame.

func NewOutputTransportMessageFrame(message any) *OutputTransportMessageFrame

func (f *OutputTransportMessageFrame) String() string

type OutputTransportMessageUrgentFrame ¶ added in v0.1.0

type OutputTransportMessageUrgentFrame struct {
	BaseSystemFrame
	// Message is the message payload to send; the transport serializes it.
	Message any
}

OutputTransportMessageUrgentFrame carries an application message that must be sent to the client immediately, ahead of any queued audio. It is a system frame; prefer OutputTransportMessageFrame when the message should stay ordered with the bot's speech.

func NewOutputTransportMessageUrgentFrame(message any) *OutputTransportMessageUrgentFrame

func (f *OutputTransportMessageUrgentFrame) String() string

type OutputTransportReadyFrame ¶ added in v0.1.0

type OutputTransportReadyFrame struct {
	BaseControlFrame
}

OutputTransportReadyFrame reports that the output transport has opened its media path and can receive frames. It is pushed upstream once the transport is ready, so a producer that must not speak into a connection that is not up yet (an avatar or video service, say) can wait for it. It is a control frame.

func NewOutputTransportReadyFrame() *OutputTransportReadyFrame

type PipelineFlushFrame ¶ added in v0.1.0

type PipelineFlushFrame struct {
	BaseControlFrame
	UninterruptibleMixin
	// Done is closed by the worker once the probe has completed its trip.
	Done chan struct{}
	// Returning reports that the probe is on its second pass downstream, after
	// having been back up to the source.
	Returning bool
	// Origin names the worker that started the flush. A probe that crosses into
	// another pipeline is answered there, out of sight of whoever is waiting, so
	// the answering worker reports progress back to this name.
	Origin string
	// contains filtered or unexported fields
}

PipelineFlushFrame is a probe that reports when the pipeline has drained.

It is pushed downstream; the worker's sink bounces it back upstream, the source turns it around, and the worker closes Done when it reaches the sink a second time. By then every frame queued ahead of it has been processed, along with anything a processor started by pushing upstream. The extra leg is what makes the probe wait for that second kind of work: an LLM run triggered by a function call result, say, whose response only comes back down after the turnaround. A probe that stopped at the source would return while that response was still being generated, or still being synthesized.

Waiting on Done therefore means the pipeline has settled, which is useful after an interruption before injecting new work.

It is uninterruptible so the probe survives an InterruptionFrame and still completes its trip; otherwise a waiter would block forever. Done is carried on the frame so concurrent flushes stay isolated, each awaiting its own probe.

func NewPipelineFlushFrame() *PipelineFlushFrame

func (f *PipelineFlushFrame) CloseDone()

type ProcessingMetricsData ¶ added in v0.1.0

type ProcessingMetricsData struct {
	BaseMetricsData
	// Value is the measured processing time.
	Value time.Duration
}

ProcessingMetricsData is the wall-clock time an operation took.

type ProcessorTarget ¶ added in v0.1.0

type ProcessorTarget interface {
	Name() string
}

ProcessorTarget identifies the processor a frame is addressed to. Like ErrorSource it is declared here rather than imported from the processor package, so the frames package keeps no dependency on it; a frame processor satisfies it by exposing its name.

type ProposedUserStartedSpeakingFrame ¶ added in v0.1.0

type ProposedUserStartedSpeakingFrame struct {
	BaseSystemFrame
}

ProposedUserStartedSpeakingFrame proposes that the user's turn has started.

It is emitted by a component with turn detection of its own, typically an STT or realtime LLM service whose provider reports speech boundaries. It is a proposal rather than a decision: an external turn-start strategy resolves it into a UserStartedSpeakingFrame and broadcasts the interruption.

It is a system frame because resolving it broadcasts an interruption, which has to preempt the frames already queued rather than wait behind them. Its end-of-turn counterpart has the opposite requirement and is a control frame; see ProposedUserStoppedSpeakingFrame.

func NewProposedUserStartedSpeakingFrame() *ProposedUserStartedSpeakingFrame

type ProposedUserStoppedSpeakingFrame ¶ added in v0.1.0

type ProposedUserStoppedSpeakingFrame struct {
	BaseControlFrame
}

ProposedUserStoppedSpeakingFrame proposes that the user's turn has ended. Like its counterpart it is a proposal an external turn-stop strategy resolves into a UserStoppedSpeakingFrame.

It is a control frame so that it stays ordered against the final TranscriptionFrame. A service with turn detection of its own pushes that transcript and then proposes the stop, and the strategy needs the text in hand to close the turn on it.

func NewProposedUserStoppedSpeakingFrame() *ProposedUserStoppedSpeakingFrame

type Role ¶

type Role string

Role identifies who authored a conversation message.

const (
	// RoleSystem is the system prompt that frames the assistant's behavior.
	RoleSystem Role = "system"
	// RoleUser is a message from the user.
	RoleUser Role = "user"
	// RoleAssistant is a message from the assistant.
	RoleAssistant Role = "assistant"
	// RoleDeveloper is an out-of-band instruction to the model. It carries the
	// results an asynchronous tool reports after its turn has moved on (see
	// AsyncToolMessage). Providers without a developer role take it as a user
	// message.
	RoleDeveloper Role = "developer"
)

type STTMetadataFrame ¶

type STTMetadataFrame struct {
	ServiceMetadataFrame
	// TTFSP99Latency is the p99 latency from end of speech to a finalized
	// transcript. Zero means unknown; strategies fall back to a default.
	TTFSP99Latency time.Duration
}

STTMetadataFrame is the metadata an STT service broadcasts. Turn-stop strategies use the p99 time-to-final-speech latency to size their safety-net timeouts, and a service that does its own server-side endpointing recommends external turn strategies through UserTurnStrategies.

func NewSTTMetadataFrame(ttfsP99 time.Duration) *STTMetadataFrame

func (f *STTMetadataFrame) String() string

type STTMuteFrame ¶ added in v0.1.0

type STTMuteFrame struct {
	BaseSystemFrame
	// Mute reports whether the service should be muted.
	Mute bool
}

STTMuteFrame mutes or unmutes the transcription service, so audio reaching it is left untranscribed for as long as the mute stands. It is a system frame, so it takes effect ahead of the audio queued behind it.

func NewSTTMuteFrame(mute bool) *STTMuteFrame

func (f *STTMuteFrame) String() string

type STTUpdateSettingsFrame ¶ added in v0.1.0

type STTUpdateSettingsFrame struct {
	ServiceUpdateSettingsFrame
}

STTUpdateSettingsFrame changes a transcription service's settings.

func NewSTTUpdateSettingsFrame(delta any) *STTUpdateSettingsFrame

func (f *STTUpdateSettingsFrame) Copy() SettingsUpdate

func (f *STTUpdateSettingsFrame) String() string

type STTUsage ¶ added in v0.1.0

type STTUsage struct {
	// AudioSeconds is the seconds of audio submitted since the last report.
	AudioSeconds float64
}

STTUsage is how much audio a speech-to-text service was given. It is raw usage, not cost, and each report is the amount since the one before it, so a consumer sums them across a session.

A streaming service is sent all the audio, silence included, so its total approximates the length of the stream, which is what most stream-priced providers bill. A segmented service is sent only the detected speech, so its total covers just those segments.

type STTUsageMetricsData ¶ added in v0.1.0

type STTUsageMetricsData struct {
	BaseMetricsData
	// Value is the usage.
	Value STTUsage
}

STTUsageMetricsData is the audio a speech-to-text service was given.

type ServiceMetadata ¶ added in v0.0.4

type ServiceMetadata interface {
	SystemFrame
	// Service is the name of the service that broadcast the metadata.
	Service() string
	// RecommendedUserTurnStrategies are the user turn strategies the service
	// recommends, or nil.
	RecommendedUserTurnStrategies() any
}

ServiceMetadata is implemented by every metadata frame a service broadcasts at pipeline start. Downstream processors assert this interface to read the common fields; a concrete STTMetadataFrame or LLMServiceMetadataFrame carries more.

type ServiceMetadataFrame ¶ added in v0.0.4

type ServiceMetadataFrame struct {
	BaseSystemFrame
	// ServiceName names the broadcasting service.
	ServiceName string
	// UserTurnStrategies are the user turn strategies the service recommends,
	// for example the external ones a service that does its own server-side
	// end-of-turn detection asks for. The user aggregator applies them unless
	// the application configured its own, which always win. Nil leaves whatever
	// is in place alone.
	//
	// It holds a turns.UserTurnStrategies. The type is not named here because
	// the turn strategies are built on this package, so naming it would be a
	// cycle.
	UserTurnStrategies any
}

ServiceMetadataFrame is broadcast by a service at pipeline start to share its configuration and performance characteristics with downstream processors. It is a system frame. STTMetadataFrame and LLMServiceMetadataFrame embed it to add their service-specific fields.

func NewServiceMetadataFrame(service string) *ServiceMetadataFrame

func (f *ServiceMetadataFrame) RecommendedUserTurnStrategies() any

func (f *ServiceMetadataFrame) Service() string

func (f *ServiceMetadataFrame) String() string

type ServiceSwitcherRequestMetadataFrame ¶ added in v0.1.0

type ServiceSwitcherRequestMetadataFrame struct {
	BaseControlFrame
	// Service is the service that should broadcast its metadata again.
	Service ServiceTarget
}

ServiceSwitcherRequestMetadataFrame asks a service to broadcast its metadata again. A switcher sends one to the service it has just made active, so what the rest of the pipeline knows describes the service now in use rather than the one it replaced. It is a control frame.

func NewServiceSwitcherRequestMetadataFrame(svc ServiceTarget) *ServiceSwitcherRequestMetadataFrame

func (f *ServiceSwitcherRequestMetadataFrame) String() string

type ServiceTarget ¶ added in v0.1.0

type ServiceTarget interface {
	Name() string
}

ServiceTarget identifies one service in the pipeline, so a settings update meant for a single service is applied by that one and forwarded untouched by the rest. It is declared here, rather than imported from the processor package, so the frames package keeps no dependency on it; a frame processor satisfies it by exposing its name.

type ServiceUpdateSettingsFrame ¶ added in v0.1.0

type ServiceUpdateSettingsFrame struct {
	BaseControlFrame
	UninterruptibleMixin

	// Delta is the settings to change, as a pointer to a settings value of the
	// kind the target service holds. Only the fields it gives are changed.
	Delta any
	// Settings names the same thing as plain data, for an update arriving from
	// outside the process with no typed value to carry. Delta wins when both are
	// set.
	Settings map[string]any
	// Service, when set, is the one service meant to apply this. Every other
	// service forwards the frame untouched. Leave it nil to reach every service
	// of the kind the frame names.
	Service ServiceTarget
	// ReachInactiveServices asks that the update reach every service a switcher
	// manages rather than only the one currently in use, for a setting that has
	// to survive a switch.
	ReachInactiveServices bool
}

ServiceUpdateSettingsFrame changes a service's settings while the pipeline runs: the voice a bot speaks in, the language it transcribes, the model it answers with. It is a control frame, so it is applied in order with the frames around it rather than jumping ahead of speech already on its way out, and it is uninterruptible, so a barge-in arriving at the same moment does not drop it.

Embed it in the frame for a kind of service rather than sending it directly.

func (f *ServiceUpdateSettingsFrame) ServiceUpdate() *ServiceUpdateSettingsFrame

func (f *ServiceUpdateSettingsFrame) TargetsService(svc ServiceTarget) bool

type SettingsUpdate ¶ added in v0.1.0

type SettingsUpdate interface {
	Frame
	// ServiceUpdate returns the update itself.
	ServiceUpdate() *ServiceUpdateSettingsFrame
	// Copy returns the same update as a new frame with an id of its own, and of
	// the same concrete kind so whatever handles the original handles the copy.
	// It is how one update is delivered to several services and still told apart
	// afterwards.
	Copy() SettingsUpdate
}

SettingsUpdate is implemented by every settings update frame, whatever kind of service it names, so a processor that only routes them can read where an update is meant to go without knowing which kind it is.

type SpeechControlParamsFrame ¶

type SpeechControlParamsFrame struct {
	BaseSystemFrame
	// VADParams are the voice-activity parameters in force, or nil.
	VADParams *vad.Params
	// TurnParams are the end-of-turn parameters in force, or nil.
	TurnParams *turn.Params
}

SpeechControlParamsFrame reports the parameters governing speech detection and end-of-turn analysis, so a processor downstream can size its own behavior to them (speech recognition matching its endpointing to the detector's, say) and clients and observers can mirror them. Either set is nil when only the other is being reported. It is a system frame.

func NewSpeechControlParamsFrame(vadParams *vad.Params, turnParams *turn.Params) *SpeechControlParamsFrame

type SpeechOutputAudioRawFrame ¶ added in v0.1.0

type SpeechOutputAudioRawFrame struct {
	OutputAudioRawFrame
}

SpeechOutputAudioRawFrame is one chunk of a continuous stream of speech audio. The stream can also carry silence between utterances, so a consumer that needs to tell the two apart has to test the samples themselves.

func NewSpeechOutputAudioRawFrame(audio []byte, sampleRate, numChannels int) *SpeechOutputAudioRawFrame

type StartFrame ¶

type StartFrame struct {
	BaseSystemFrame

	// AudioInSampleRate is the input audio sample rate in Hz.
	AudioInSampleRate int
	// AudioOutSampleRate is the output audio sample rate in Hz.
	AudioOutSampleRate int
	// EnableMetrics enables performance metrics collection.
	EnableMetrics bool
	// EnableUsageMetrics enables usage metrics collection.
	EnableUsageMetrics bool
	// ReportOnlyInitialTTFB reports only the initial time-to-first-byte.
	ReportOnlyInitialTTFB bool
}

StartFrame is the first frame pushed down a pipeline. It initializes every processor with the pipeline-wide configuration. It is a system frame.

func NewStartFrame() *StartFrame

type StopFrame ¶ added in v0.1.0

type StopFrame struct {
	BaseControlFrame
	UninterruptibleMixin
}

StopFrame indicates the pipeline should stop but that its processors are to be kept running, ready for another run. Unlike EndFrame it does not shut processors down. It is normally queued by the Task. It is uninterruptible so a barge-in cannot drop it.

func NewStopFrame() *StopFrame

type StopWorkerFrame ¶ added in v0.1.0

type StopWorkerFrame struct {
	BaseControlFrame
	UninterruptibleMixin
}

StopWorkerFrame requests that the Task stop once queued frames are flushed, while leaving the processors running and ready for another run. On reaching the Task it queues a StopFrame. It is the counterpart of EndWorkerFrame for a run that should stop without shutting processors down.

func NewStopWorkerFrame() *StopWorkerFrame

type SummaryConfig ¶ added in v0.1.0

type SummaryConfig struct {
	// TargetContextTokens caps the length of the generated summary, passed to
	// the model as its token limit. Size it so the summary plus the messages
	// kept after it still fit the model's context comfortably. Zero uses 6000.
	TargetContextTokens int
	// MinMessagesAfterSummary is how many of the most recent messages are kept
	// out of the summary, so the immediate conversational context survives
	// compression verbatim. Zero uses 4; a negative value is rejected.
	MinMessagesAfterSummary int
	// SummarizationPrompt tells the model how to summarize. Empty uses
	// DefaultSummarizationPrompt.
	SummarizationPrompt string
	// SummaryMessageTemplate formats the summary as it is written back into the
	// conversation. It must contain "{summary}", which is replaced by the
	// generated text, so an application can wrap the summary in delimiters of
	// its own and let the system prompt tell a summary from live conversation.
	// Empty uses "Conversation summary: {summary}".
	SummaryMessageTemplate string
	// LLM generates the summary on its own, instead of the conversation's LLM.
	// Set it to route summarization to a cheaper or faster model while an
	// expensive one carries the conversation. Nil sends the request to the
	// pipeline's LLM instead, as an LLMContextSummaryRequestFrame.
	//
	// It is typed as any because the LLM services are built on this package, so
	// their interface cannot be named here. Set it to an llm.Inferencer;
	// anything else is reported and the request goes to the pipeline's LLM.
	LLM any
	// SummarizationTimeout bounds one summary generation. Exceeding it abandons
	// that summary and unblocks the next. Zero uses DefaultSummarizationTimeout.
	SummarizationTimeout time.Duration
}

SummaryConfig controls how a summary is generated. It is shared by automatic and on-demand summarization alike.

func (c SummaryConfig) SummaryPrompt() string

func (c SummaryConfig) Validate() error

func (c SummaryConfig) WithDefaults() SummaryConfig

type SwitcherFrame ¶ added in v0.1.0

type SwitcherFrame interface {
	Frame
	// contains filtered or unexported methods
}

SwitcherFrame is implemented by the frames that steer a service switcher. A switcher hands one to its strategy, which decides what to do with it; a frame the strategy does not act on travels on, so a switcher further down the pipeline gets its turn.

type SystemFrame ¶

type SystemFrame interface {
	Frame
	// contains filtered or unexported methods
}

SystemFrame takes priority over other frames and is not affected by user interruptions; system frames are handled in order. Assert a Frame to SystemFrame to test its category. Embed BaseSystemFrame to define one.

type TTFAMetricsData ¶ added in v0.1.0

type TTFAMetricsData struct {
	BaseMetricsData
	// TTFA is the time to the first audible sample: TTFB plus LeadingSilence.
	TTFA time.Duration
	// TTFB is the time to first byte that TTFA builds on.
	TTFB time.Duration
	// LeadingSilence is the silence before the first audible sample: TTFA minus
	// TTFB.
	LeadingSilence time.Duration
}

TTFAMetricsData is time to first audible sample: time to first byte plus any silence a TTS service padded onto the start of its response.

It is reported with its breakdown so a consumer can see how much of the delay the listener hears is padding rather than the service answering, without having to match it up with the TTFBMetricsData reported separately. TTFB here is that same measurement, not another one, so do not add the two together.

type TTFBMetricsData ¶ added in v0.1.0

type TTFBMetricsData struct {
	BaseMetricsData
	// Value is the measured time to first byte.
	Value time.Duration
}

TTFBMetricsData is time to first byte: how long a service took to produce anything at all, its first token or its first audio.

type TTSAudioRawFrame ¶

type TTSAudioRawFrame struct {
	OutputAudioRawFrame
	// ContextID identifies the TTS context that generated this audio; "" when
	// unset.
	ContextID string
}

TTSAudioRawFrame is a chunk of output audio generated by a TTS service, ready for playback.

func NewTTSAudioRawFrame(audio []byte, sampleRate, numChannels int) *TTSAudioRawFrame

type TTSSpeakFrame ¶

type TTSSpeakFrame struct {
	BaseDataFrame
	// Text is the exact text to speak.
	Text string
	// AppendToContext reports whether the spoken text is appended to the LLM
	// context as an assistant message. Defaults to true; set it false for
	// utterances that should not become part of the conversation (e.g. a wake
	// acknowledgement, which would otherwise start the context on an assistant
	// turn).
	AppendToContext bool
}

TTSSpeakFrame carries fixed text for the TTS service to speak directly, bypassing the LLM and the TTS sentence aggregator — the way to make the bot say a set phrase (a greeting, an acknowledgement). It is a data frame.

func NewTTSSpeakFrame(text string) *TTSSpeakFrame

func (f *TTSSpeakFrame) String() string

type TTSStartedFrame ¶

type TTSStartedFrame struct {
	BaseControlFrame
	// ContextID identifies this TTS context; "" when unset.
	ContextID string
	// AppendToContext reports whether the spoken text for this response will be
	// appended to the LLM context. Defaults to true.
	AppendToContext bool
}

TTSStartedFrame marks the beginning of a TTS response. The following TTSAudioRawFrames are part of the response until a TTSStoppedFrame. It is a control frame.

func NewTTSStartedFrame() *TTSStartedFrame

type TTSStoppedFrame ¶

type TTSStoppedFrame struct {
	BaseControlFrame
	// ContextID identifies this TTS context; "" when unset.
	ContextID string
}

TTSStoppedFrame marks the end of a TTS response. It is a control frame.

func NewTTSStoppedFrame() *TTSStoppedFrame

type TTSTextFrame ¶ added in v0.1.0

type TTSTextFrame struct {
	TextFrame
	// RawText is the original written form of this span; "" means use Text.
	RawText string
	// ContextID identifies the TTS context that produced this text; "" when unset.
	ContextID string
}

TTSTextFrame is a chunk of text a TTS service is speaking, aligned to audio playback. Text is the token as it was sent to the synthesizer; RawText, when set, is that same span in its original written form (for example "$42.50" for a token spoken as "forty two dollars and fifty cents"), so the assistant context records what was written rather than what was pronounced. A TTS service that reports word timings emits one per spoken word as its audio plays; because they flow in step with playback, an interruption leaves only the frames already emitted — the words actually spoken — in the context.

func NewTTSTextFrame(text string) *TTSTextFrame

func (f *TTSTextFrame) Original() string

func (f *TTSTextFrame) String() string

type TTSUpdateSettingsFrame ¶ added in v0.1.0

type TTSUpdateSettingsFrame struct {
	ServiceUpdateSettingsFrame
}

TTSUpdateSettingsFrame changes a speech synthesis service's settings.

func NewTTSUpdateSettingsFrame(delta any) *TTSUpdateSettingsFrame

func (f *TTSUpdateSettingsFrame) Copy() SettingsUpdate

func (f *TTSUpdateSettingsFrame) String() string

type TTSUsageMetricsData ¶ added in v0.1.0

type TTSUsageMetricsData struct {
	BaseMetricsData
	// Value is the number of characters.
	Value int
}

TTSUsageMetricsData is the number of characters a TTS service synthesized, which is what providers bill against.

type TextAggregationMetricsData ¶ added in v0.1.0

type TextAggregationMetricsData struct {
	BaseMetricsData
	// Value is the measured aggregation time.
	Value time.Duration
}

TextAggregationMetricsData is the time from a model's first token to the first complete sentence: what grouping text into sentences costs before synthesis can start.

type TextFrame ¶

type TextFrame struct {
	BaseDataFrame
	// Text is the text content.
	Text string
	// SkipTTS reports whether a TTS service should skip this text. A nil value
	// means "unset": the decision is left to the frame flow.
	SkipTTS *bool
	// IncludesInterFrameSpaces reports whether any leading/trailing spaces
	// needed between adjacent frames are already part of Text.
	IncludesInterFrameSpaces bool
	// AppendToContext reports whether this text should be appended to the LLM
	// context. Defaults to true.
	AppendToContext bool
}

TextFrame is a chunk of text flowing through the pipeline — emitted by LLM services and consumed by aggregators, TTS services and more. It is a data frame.

func NewTextFrame(text string) *TextFrame

func (f *TextFrame) String() string

type Tool ¶

type Tool struct {
	Name        string
	Description string
	Parameters  json.RawMessage
	// Handler runs the tool. Setting it means the tool carries its own
	// implementation: the LLM service registers it when the toolset is
	// advertised and drops it again when the toolset stops advertising it, so
	// what the model can call and what actually answers stay the same thing.
	//
	// A tool with no Handler is advertise-only, and something must have called
	// RegisterFunction for it. Registering explicitly always wins: a handler
	// registered by hand is never replaced by a tool's own, and never dropped.
	//
	// It is typed as any because the handler signature belongs to the LLM
	// service, which is built on this package. Set it to an llm.FunctionCallHandler;
	// anything else is reported and ignored.
	Handler any
	// Cleanup names a resource this tool works through that outlives the calls it
	// serves, a connection to a tool server being the case that matters. The LLM
	// service releases it when the pipeline tears down, so registering the tool is
	// all it takes to have the connection closed as well. Tools sharing one
	// resource name the same value and it is released once.
	//
	// It is typed as any for the same reason Handler is: what a resource has to
	// implement belongs to the LLM service, which is built on this package. Set
	// it to an llm.ToolCleanup; anything else is reported and ignored.
	Cleanup any
	// CancelOnInterruption sets whether a call to this tool is canceled when the
	// user interrupts. Nil leaves the service's default, which cancels. It is
	// only read for a tool that carries its own Handler: a handler registered by
	// hand carries the options its own registration gave it.
	//
	// A tool that is not canceled is asynchronous: the conversation continues
	// without waiting for it, and its result is delivered whenever it arrives.
	CancelOnInterruption *bool
	// TimeoutSecs bounds how long a call to this tool may take, overriding the
	// service-wide bound. Nil leaves the service-wide one. Like
	// CancelOnInterruption it is only read for a tool that carries a Handler.
	TimeoutSecs *float64
	// CancellableByLLM says whether the model may cancel a running call of this
	// tool, which advertises a cancel tool named for it. Nil leaves the service's
	// default, which is not to. It is only meaningful on an asynchronous tool,
	// and like the two above it is only read for a tool that carries a Handler.
	CancellableByLLM *bool
}

Tool is a function the model may call. Parameters is a JSON-Schema object (`{"type":"object","properties":{…},"required":[…]}`) describing the arguments the tool accepts.

type ToolCall ¶

type ToolCall struct {
	ID   string
	Name string
	Args json.RawMessage
}

ToolCall is a request from the model to invoke a tool. Args is the raw JSON arguments the model produced.

type ToolChoice ¶ added in v0.1.0

type ToolChoice string

ToolChoice tells the model whether it may or must call a tool.

const (
	// ToolChoiceAuto lets the model decide whether to call a tool.
	ToolChoiceAuto ToolChoice = "auto"
	// ToolChoiceNone forbids tool calls.
	ToolChoiceNone ToolChoice = "none"
	// ToolChoiceRequired requires the model to call a tool.
	ToolChoiceRequired ToolChoice = "required"
)

type ToolResult ¶

type ToolResult struct {
	ID      string
	Name    string
	Content string
}

ToolResult is the outcome of a tool invocation, paired to a ToolCall by ID.

type ToolsSchema ¶ added in v0.1.0

type ToolsSchema struct {
	Standard []Tool
	Custom   map[AdapterType][]any
}

ToolsSchema is the toolset a conversation advertises.

Standard holds the tools every provider is offered, described the same way for all of them. Custom holds tools written in one provider's own format, for the ones no common description fits: a hosted search tool, say, which the provider implements itself rather than calling back for. A provider is only ever sent the custom tools written for its own format, so a conversation carrying them is still usable with every other provider.

Each custom tool must be the tool type its adapter's package defines. Not every provider takes them: Anthropic has no custom tools, and anything under a key it does not read is left out.

func (s ToolsSchema) CustomFor(t AdapterType) []any

type TranscriptionFrame ¶

type TranscriptionFrame struct {
	TextFrame
	// UserID identifies the user who spoke.
	UserID string
	// Timestamp is when the transcription occurred.
	Timestamp string
	// Language is the detected or specified language as a BCP-47 tag; "" when
	// unset.
	Language string
	// Result is the raw result from the STT service, if available.
	Result any
	// Finalized reports whether this is the final transcription for an
	// utterance, for STT services that signal commit/finalize.
	Finalized bool
}

TranscriptionFrame carries a finalized speech transcription for a user.

func NewTranscriptionFrame(text, userID, timestamp string) *TranscriptionFrame

func (f *TranscriptionFrame) String() string

type TranslationFrame ¶ added in v0.1.0

type TranslationFrame struct {
	TextFrame
	// UserID identifies the user who spoke.
	UserID string
	// Timestamp is when the translation occurred.
	Timestamp string
	// Language is the language translated into, as a BCP-47 tag; "" when unset.
	Language string
}

TranslationFrame carries a translated transcription for a user, distinct from the transcription it was translated from: a provider that transcribes and translates reports both, and only the transcription is the user's own words.

func NewTranslationFrame(text, userID, timestamp string) *TranslationFrame

func (f *TranslationFrame) String() string

type TurnMetricsData ¶ added in v0.1.0

type TurnMetricsData struct {
	BaseMetricsData
	// Complete is whether the turn was predicted to be finished.
	Complete bool
	// Probability is the analyzer's confidence that it was finished.
	Probability float64
	// E2EProcessing is how long deciding took end to end, measured from the
	// point speech turned to silence.
	E2EProcessing time.Duration
}

TurnMetricsData is what an end-of-turn analyzer decided about a user turn, and what it cost to decide. Without it a turn that ends on the safety-net timeout is indistinguishable from one the analyzer judged unfinished.

type Uninterruptible ¶

type Uninterruptible interface {
	// contains filtered or unexported methods
}

Uninterruptible marks a data or control frame that must survive interruptions: it stays queued and any task processing it is never canceled, guaranteeing delivery and completion. Embed UninterruptibleMixin (alongside a category base) and assert with this interface.

type UninterruptibleMixin ¶

type UninterruptibleMixin struct{}

UninterruptibleMixin is embedded to mark a frame Uninterruptible.

type UserAudioRawFrame ¶ added in v0.1.0

type UserAudioRawFrame struct {
	InputAudioRawFrame
	// UserID identifies the user this audio came from.
	UserID string
}

UserAudioRawFrame is a chunk of input audio attributed to the user who spoke it, for a transport carrying several participants. Input audio that names nobody is an InputAudioRawFrame.

func NewUserAudioRawFrame(userID string, audio []byte, sampleRate, numChannels int) *UserAudioRawFrame

func (f *UserAudioRawFrame) String() string

type UserIdleTimeoutUpdateFrame ¶

type UserIdleTimeoutUpdateFrame struct {
	BaseSystemFrame
	// Timeout is the new idle timeout.
	Timeout time.Duration
}

UserIdleTimeoutUpdateFrame updates the user-idle timeout at runtime. A value <= 0 disables idle detection. It is a system frame.

func NewUserIdleTimeoutUpdateFrame(timeout time.Duration) *UserIdleTimeoutUpdateFrame

func (f *UserIdleTimeoutUpdateFrame) String() string

type UserMuteStartedFrame ¶

type UserMuteStartedFrame struct {
	BaseSystemFrame
}

UserMuteStartedFrame reports that user input is now being suppressed (a mute strategy engaged). It is a system frame.

func NewUserMuteStartedFrame() *UserMuteStartedFrame

type UserMuteStoppedFrame ¶

type UserMuteStoppedFrame struct {
	BaseSystemFrame
}

UserMuteStoppedFrame reports that user input is no longer suppressed. It is a system frame.

func NewUserMuteStoppedFrame() *UserMuteStoppedFrame

type UserSpeakingFrame ¶

type UserSpeakingFrame struct {
	BaseSystemFrame
}

UserSpeakingFrame is emitted periodically while the user is speaking, a keepalive that lets strategies and idle logic know audio is still arriving. It is a system frame.

func NewUserSpeakingFrame() *UserSpeakingFrame

type UserStartedSpeakingFrame ¶

type UserStartedSpeakingFrame struct {
	BaseSystemFrame
}

UserStartedSpeakingFrame indicates the user turn has started. It is a system frame.

func NewUserStartedSpeakingFrame() *UserStartedSpeakingFrame

type UserStoppedSpeakingFrame ¶

type UserStoppedSpeakingFrame struct {
	BaseSystemFrame
}

UserStoppedSpeakingFrame indicates the user turn has ended. It is a system frame.

func NewUserStoppedSpeakingFrame() *UserStoppedSpeakingFrame

type UserTurnInferenceCompletedFrame ¶

type UserTurnInferenceCompletedFrame struct {
	BaseControlFrame
}

UserTurnInferenceCompletedFrame signals that an external judge (an LLM completion gate, an EOT classifier) decided the user's turn is semantically complete. A turn-stop strategy waits for it to finalize the turn. It is a control frame.

func NewUserTurnInferenceCompletedFrame() *UserTurnInferenceCompletedFrame

type VADParamsUpdateFrame ¶ added in v0.1.0

type VADParamsUpdateFrame struct {
	BaseControlFrame
	// Params are the detection parameters to adopt.
	Params vad.Params
}

VADParamsUpdateFrame asks the voice-activity detector to adopt new parameters. It is pushed upstream (by the RTVI processor acting on a client request, say) and takes effect from the next chunk analyzed. It is a control frame.

func NewVADParamsUpdateFrame(params vad.Params) *VADParamsUpdateFrame

type VADUserStartedSpeakingFrame ¶

type VADUserStartedSpeakingFrame struct {
	BaseSystemFrame
	// StartSecs is the VAD's confirmation delay (how long speech persisted
	// before onset was confirmed), in seconds.
	StartSecs float64
	// Timestamp is the wall-clock time at which the VAD made its determination.
	// Subtracting StartSecs from it gives the moment the speech itself began,
	// which is what a measurement of the whole utterance is anchored to.
	Timestamp time.Time
}

VADUserStartedSpeakingFrame reports that a voice-activity detector heard the user start speaking. It is the raw VAD signal the turn subsystem consumes to decide a user turn; it is distinct from UserStartedSpeakingFrame, which is the turn decision. It is a system frame.

func NewVADUserStartedSpeakingFrame(
	startSecs float64, timestamp time.Time,
) *VADUserStartedSpeakingFrame

func (f *VADUserStartedSpeakingFrame) SpeechStart() time.Time

func (f *VADUserStartedSpeakingFrame) String() string

type VADUserStoppedSpeakingFrame ¶

type VADUserStoppedSpeakingFrame struct {
	BaseSystemFrame
	// StopSecs is the silence duration the VAD required before confirming the
	// stop, in seconds.
	StopSecs float64
	// Timestamp is the wall-clock time at which the VAD made its determination.
	// Subtracting StopSecs from it gives the moment the speech itself ended,
	// which is what a deadline measured from the end of speech is anchored to.
	// The zero value means unset.
	Timestamp time.Time
}

VADUserStoppedSpeakingFrame reports that the VAD heard the user stop speaking. It is a system frame.

func NewVADUserStoppedSpeakingFrame(stopSecs float64, timestamp time.Time) *VADUserStoppedSpeakingFrame

func (f *VADUserStoppedSpeakingFrame) String() string