ingest

type Config struct {
	// ListenAddr is the address to listen on (e.g., ":8082")
	ListenAddr string

	// Project is the git repository or project name to tag on captured turns.
	Project string
}

type MalformedStamp struct {
	Field string
	Value string
	Err   error
}

type Metrics struct {
	// contains filtered or unexported fields
}

type RawReduction struct {
	// Response is the reduced turn. Non-nil whenever the error is nil.
	Response *llm.ChatResponse

	// Truncated reports that the stored body ended early and was salvaged.
	// The reduction went ahead — a turn recovered from a stream that ended
	// early is worth more than no turn — but it may be missing its tail, and
	// nothing downstream can tell that from the row alone.
	Truncated bool

	// DurationSource and CreatedAtSource name the meta field that supplied
	// each capture-side stamp, or StampSourceFallback when the envelope
	// carried nothing usable.
	DurationSource  StampSource
	CreatedAtSource StampSource

	// MalformedStamps lists capture-side timestamps that were present but
	// unparseable.
	MalformedStamps []MalformedStamp
}

type Result string

type Server struct {
	// contains filtered or unexported fields
}

type StampField string

type StampSource string

type StoredRawTurn struct {
	// Provider keys the reducer table.
	Provider string

	// RawRequest is the original provider request body. Both current
	// reducers discard it; it is passed through because the Reducer contract
	// admits enriching a response from request context, and a reducer that
	// started doing so should see the same input on both paths.
	RawRequest json.RawMessage

	// RawResponse is the upstream response body exactly as it arrived,
	// still under RawResponseEncoding.
	RawResponse []byte

	// RawResponseEncoding is the Content-Encoding the bytes are stored
	// under. Empty means identity.
	RawResponseEncoding string

	// Meta is the capture adapter's metadata block. ContentType selects the
	// streaming vs one-shot reduction path; the timestamp and elapsed fields
	// supply the capture-side stamps the bytes cannot carry.
	Meta TurnMeta
}

type TranscriptPayload struct {
	// Session identifies the harness session the transcript belongs to.
	Session *sessions.IngestEnvelope `json:"session"`

	// AgentID is empty for the main transcript, or the subagent id for
	// subagents/agent-<id>.jsonl files.
	AgentID string `json:"agent_id,omitempty"`

	// AgentType / Description / ToolUseID mirror the harness's
	// subagent meta.json: ToolUseID is the Task tool_use that forked
	// this agent — the causal fork edge the deriver attaches.
	AgentType   string `json:"agent_type,omitempty"`
	Description string `json:"description,omitempty"`
	ToolUseID   string `json:"tool_use_id,omitempty"`

	// Kind qualifies Codex sub_agent_activity anchor rows:
	// "interacted" marks a re-entry record (send_message /
	// followup_task, targeting AgentID with ToolUseID = the triggering
	// call), banked for future rendering and ignored by derivation.
	// Absent/empty means spawn evidence — the legacy default.
	Kind string `json:"kind,omitempty"`

	// Records is the transcript's JSONL content as a JSON array,
	// verbatim.
	Records json.RawMessage `json:"records"`
}

type TurnMeta struct {
	RequestID         string `json:"request_id,omitempty"`
	UpstreamRequestID string `json:"upstream_request_id,omitempty"`
	ContentType       string `json:"content_type,omitempty"`

	// ThreadID is the harness sub-thread id resolved by the capture
	// adapter (extproc headers.ThreadID); "" for main-thread calls.
	ThreadID string `json:"thread_id,omitempty"`

	Method              string  `json:"method,omitempty"`
	Path                string  `json:"path,omitempty"`
	Endpoint            string  `json:"endpoint,omitempty"`
	Model               string  `json:"model,omitempty"`
	ModelFamily         string  `json:"model_family,omitempty"`
	Stream              string  `json:"stream,omitempty"`
	ContentEncoding     string  `json:"content_encoding,omitempty"`
	UpstreamStatus      int     `json:"upstream_status,omitempty"`
	UpstreamStatusClass string  `json:"upstream_status_class,omitempty"`
	RequestBytes        int     `json:"request_bytes,omitempty"`
	ResponseBytes       int     `json:"response_bytes,omitempty"`
	ElapsedSeconds      float64 `json:"elapsed_seconds,omitempty"`

	// TsRequest is the capture-side instant the turn's request went
	// upstream, RFC 3339. It is not new: pkg/backfill emits it and
	// derive.CapturedAt already reads it as a row's original capture
	// time, which is where every derived span's StartedAt comes from.
	// Ingest parses it here so a server-side reduction can date itself
	// from the same source the deriver uses, rather than a second,
	// silently different clock.
	TsRequest string `json:"ts_request,omitempty"`

	// CapturedAt is the capture-side instant the turn COMPLETED
	// upstream, RFC 3339 — the quantity CreatedAt actually means, and
	// the one a producer that reduces live records as time.Now().
	//
	// Distinct from TsRequest by exactly the call's duration. Preferred
	// over it because it needs no arithmetic to be exact; optional, and
	// no released producer sends it yet. See stampCaptureTime for the
	// precedence and what happens when neither field is present.
	CapturedAt string `json:"captured_at,omitempty"`
}

type TurnPayload struct {
	// Provider type: "openai", "anthropic", "ollama"
	Provider string `json:"provider"`

	// AgentName optionally tags the turn (same as X-Tapes-Agent-Name header)
	AgentName string `json:"agent_name,omitempty"`

	// RawRequest is the original request body sent to the LLM provider.
	RawRequest json.RawMessage `json:"request"`

	// Response is the already reduced, provider-agnostic response for the turn.
	Response llm.ChatResponse `json:"response"`

	// RawResponse is the upstream response body exactly as it arrived on the
	// wire, base64-encoded in the JSON envelope. Optional and independent of
	// Response: an adapter may send both (reduction plus the bytes it reduced
	// from), only the reduction (the historical shape), or only the bytes.
	//
	// Raw-only is the interesting case. Reduction is lossy and adapter-
	// specific, so an adapter that ships only the bytes lets ingest perform
	// the reduction with the shared pkg/capture reducers — which is what makes
	// two capture paths produce identical rows for identical upstream traffic
	// instead of two subtly different ones.
	RawResponse []byte `json:"raw_response,omitempty"`

	// RawResponseEncoding is the Content-Encoding of RawResponse ("identity",
	// "gzip", …). Empty means identity. The bytes are stored under this
	// encoding rather than decompressed, so the stored column stays literally
	// what the upstream sent.
	RawResponseEncoding string `json:"raw_response_encoding,omitempty"`

	// RawResponseWithheld says the producer captured verbatim bytes and
	// deliberately did not send them — almost always because including them
	// would have pushed the envelope past MaxIngestBodyBytes, so the choice
	// was between a turn without its bytes and no turn at all.
	//
	// Without it that turn is indistinguishable from one produced by an
	// adapter that never captured raw bytes in the first place: both arrive
	// with raw_response absent. Those are opposite facts. The first is a
	// limit that bit and wants tuning; the second is a deployment fact about
	// which producers are running. A fidelity report that cannot separate
	// them reports the wrong one — silently, and in the reassuring
	// direction, since "this producer doesn't send raw" reads as expected
	// where "we lost bytes we had" does not.
	//
	// Deliberately NOT named to match the raw_response_dropped column it
	// feeds. The column is the union of two causes — the producer withheld,
	// or ingest capped — and this field is only one of them. A shared name
	// would imply the producer sets the column, when it contributes to it.
	//
	// Absent means "no claim", which is exactly the pre-existing behavior:
	// every producer shipped before this field omits it and keeps reading as
	// it always did.
	RawResponseWithheld bool `json:"raw_response_withheld,omitempty"`

	// Meta is the capture adapter's metadata block. Parsed for the
	// fields ingest promotes (request IDs and raw-turn dedup); the
	// verbatim JSON is persisted alongside the raw turn so fields
	// unknown to this build survive.
	Meta TurnMeta `json:"meta"`

	// Session is the optional session-tracking envelope. When present,
	// ingest UPSERTs a `sessions` row keyed by
	// (org_id, harness_id, harness_session_id), resolves the
	// parent_session_id FK (placeholder-inserting when needed), and
	// rolls up turn counters — all in the same transaction as the
	// nodes insert. When absent, ingest treats the turn as
	// harness_id="unknown" and derives a synthetic harness_session_id
	// from the captured turn's Merkle root prefix.
	//
	// The type lives in pkg/sessions to avoid an import cycle
	// (proxy/worker depends on it too).
	Session *sessions.IngestEnvelope `json:"session,omitempty"`
}