computeruse

type Action struct {
	// CallID identifies the model call the action belongs to; the result
	// of the action carries the same ID.
	CallID string `json:"call_id,omitempty"`
	Kind   Kind   `json:"kind"`
	Point  *Point `json:"point,omitempty"`
	// ElementID names an element from the observation to act on. The
	// executor resolves it to a point before the action runs.
	ElementID string        `json:"element_id,omitempty"`
	Path      []Point       `json:"path,omitempty"`
	Button    string        `json:"button,omitempty"`
	Count     int           `json:"count,omitempty"`
	Modifiers []string      `json:"modifiers,omitempty"`
	Text      string        `json:"text,omitempty"`
	Keys      []string      `json:"keys,omitempty"`
	Repeat    int           `json:"repeat,omitempty"`
	ScrollX   int           `json:"scroll_x,omitempty"`
	ScrollY   int           `json:"scroll_y,omitempty"`
	Duration  time.Duration `json:"duration,omitempty"`
	Region    *Rect         `json:"region,omitempty"`
}

type Done struct {
	Success bool   `json:"success"`
	Summary string `json:"summary"`
}

type Element struct {
	ID   string
	Role string
	Name string
	// Value is what the element holds, such as a field's text or the option
	// a combo box shows, with the values the step types written as their
	// placeholders.
	Value string
	// Hint is the text an empty field shows in place of a value, such as a
	// grey "0", which the field does not hold. Empty when the field holds a
	// value or the desktop does not report one.
	Hint string
}

type ImageLimit struct {
	LongEdge  int
	MaxPixels int
}

type Kind string

type Mode string

type NativeFactory func(provider llm.Provider, opts Options) (Session, error)

type Observation struct {
	// Screen is the current screen.
	Screen Screen
	// Results answer the previous turn's actions, one per action, in order.
	Results []Result
	// Acknowledged approves the previous turn's Confirmation.
	Acknowledged bool
	// Note is an instruction for the model, such as a reminder to report
	// the task done.
	Note string
	// ReportOnly asks the model for its report instead of more actions. A
	// session that can limit the model's tools offers it only done.
	ReportOnly bool
	// Elements are the actionable elements of the front window the model
	// may target by id, more reliable than aiming at a pixel. Empty when
	// the desktop exposes no elements.
	Elements []Element
}

type Options struct {
	// Model is the model name.
	Model string
	// Task is what the model should accomplish.
	Task string
	// System adds instructions about the environment, such as the
	// operating system.
	System string
	// MaxTokens caps each model response.
	MaxTokens *int
	// Temperature applies to models that accept it.
	Temperature *float64
}

type Point struct {
	X int `json:"x"`
	Y int `json:"y"`
}

type Rect struct {
	Min Point `json:"min"`
	Max Point `json:"max"`
}

type Result struct {
	CallID string
	// Output is text the action produced, such as a cursor position.
	Output string
	// Image is the picture a screenshot or zoom action produced.
	Image *llm.Image
	// Error describes why the action failed.
	Error string
	// Skipped marks an action not performed because an earlier one failed.
	Skipped bool
}

type Screen struct {
	Image  llm.Image
	Width  int
	Height int
}

type Session interface {
	// ImageLimit reports the largest screenshot the session accepts.
	ImageLimit() ImageLimit
	// Next sends an observation and returns the model's next turn.
	Next(ctx context.Context, obs Observation) (*Turn, error)
}

type Turn struct {
	// Actions are performed in order, stopping at the first failure.
	Actions []Action
	// Done is set when the model finished the task; any Actions of the
	// same turn are performed first.
	Done *Done
	// Confirmation, when set, is the model provider's explanation of why
	// Actions need a person's approval before they run.
	Confirmation string
	// Text is what the model wrote alongside its actions.
	Text string
	// Usage counts the tokens of this turn.
	Usage llm.Usage
}