sources

package
v0.16.5 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 4, 2026 License: AGPL-3.0 Imports: 16 Imported by: 0

README

sources

Package sources provides abstractions for fetching AI model catalog data from various external sources including provider APIs and community repositories.

sources

import "github.com/agentstation/starmap/pkg/sources"

Package sources provides public APIs for working with AI model data sources.

Index

Constants

const (
    // RevisionKindUnknown means the upstream exposes no stable revision.
    RevisionKindUnknown = evidence.ObservationRevisionKindUnknown
    // RevisionKindETag identifies an HTTP entity-tag revision.
    RevisionKindETag = evidence.ObservationRevisionKindETag
    // RevisionKindLastModified identifies an HTTP Last-Modified validator.
    RevisionKindLastModified = evidence.ObservationRevisionKindLastModified
    // RevisionKindGitCommit identifies an exact Git commit.
    RevisionKindGitCommit = evidence.ObservationRevisionKindGitCommit
    // RevisionKindSourceVersion identifies an upstream-declared version.
    RevisionKindSourceVersion = evidence.ObservationRevisionKindSourceVersion
    // RevisionKindContentDigest identifies the normalized observation content.
    RevisionKindContentDigest = evidence.ObservationRevisionKindContentDigest
)

const (
    // ObservationCompletenessComplete means the observation contains every expected record.
    ObservationCompletenessComplete = evidence.ObservationCompletenessComplete
    // ObservationCompletenessPartial means at least one expected record is absent.
    ObservationCompletenessPartial = evidence.ObservationCompletenessPartial
)

const (
    // ObservationStatusSucceeded means the observation completed without known degradation.
    ObservationStatusSucceeded = evidence.ObservationStatusSucceeded
    // ObservationStatusDegraded means the observation contains known issues.
    ObservationStatusDegraded = evidence.ObservationStatusDegraded
)

Observation issue scope values.

const (
    ObservationIssueScopeRecord        = evidence.ObservationIssueScopeRecord
    ObservationIssueScopeProvider      = evidence.ObservationIssueScopeProvider
    ObservationIssueScopeSource        = evidence.ObservationIssueScopeSource
    ObservationIssueScopeStaleFallback = evidence.ObservationIssueScopeStaleFallback
)

Observation issue code values.

const (
    ObservationIssueCodeInvalidRecord      = evidence.ObservationIssueCodeInvalidRecord
    ObservationIssueCodeSchemaDrift        = evidence.ObservationIssueCodeSchemaDrift
    ObservationIssueCodePayloadLimit       = evidence.ObservationIssueCodePayloadLimit
    ObservationIssueCodeMissingCredentials = evidence.ObservationIssueCodeMissingCredentials
    ObservationIssueCodeConfiguration      = evidence.ObservationIssueCodeConfiguration
    ObservationIssueCodeFetchFailed        = evidence.ObservationIssueCodeFetchFailed
    ObservationIssueCodeStaleFallback      = evidence.ObservationIssueCodeStaleFallback
    ObservationIssueCodeBootstrapFallback  = evidence.ObservationIssueCodeBootstrapFallback
    ObservationIssueCodeVolumeCollapse     = evidence.ObservationIssueCodeVolumeCollapse
)

Source identifiers use names that read clearly at the Source API.

const (
    ProvidersID       = evidence.ProvidersID
    ModelsDevGitID    = evidence.ModelsDevGitID
    ModelsDevHTTPID   = evidence.ModelsDevHTTPID
    LocalCatalogID    = evidence.LocalCatalogID
    ReleaseArtifactID = evidence.ReleaseArtifactID
    EmbeddedCatalogID = evidence.EmbeddedCatalogID
)

MaxJSONNestingDepth bounds object/array nesting before JSON decode.

const MaxJSONNestingDepth = sourcepayload.MaxJSONNestingDepth

func ValidateJSONPayload

func ValidateJSONPayload(data []byte) error

ValidateJSONPayload enforces source byte and nesting limits before decoding.

type Dependency

Dependency represents an external tool or runtime required by a source.

type Dependency struct {
    // Core identification
    Name        string // Machine name: "bun", "git", "docker"
    DisplayName string // Human-readable: "Bun JavaScript runtime"
    Required    bool   // false = source is optional or has fallback

    // Checking availability
    CheckCommands []string // Try in order: ["bun", "bunx"]
    MinVersion    string   // Optional: "1.0.0"

    // Installation
    InstallURL         string // https://bun.sh/docs/installation
    AutoInstallCommand string // Optional: "curl -fsSL https://bun.sh/install | bash"

    // User messaging
    Description       string // "Builds models.dev data locally (same as HTTP source)"
    WhyNeeded         string // "Required to build api.json from TypeScript source"
    AlternativeSource string // "models_dev_http provides same data without dependencies"
}

type DependencyStatus

DependencyStatus represents the availability status of a dependency.

type DependencyStatus struct {
    Available  bool   // Whether the dependency is available
    Version    string // Version string if available and detectable
    Path       string // Full path to executable if found
    CheckError error  // Error from check command if not available
}

type FetchStats

FetchStats contains metadata about a fetch operation. This provides transparency into API requests for debugging and monitoring.

type FetchStats struct {
    URL          string
    StatusCode   int           // HTTP response status code
    Latency      time.Duration // Request duration
    PayloadSize  int64         // Response body size in bytes
    ContentType  string        // Content-Type from response header
    AuthMethod   string
    AuthLocation string
    AuthScheme   string // Authentication scheme for header auth (Bearer, Basic, Direct)
}

func (*FetchStats) HumanSize
func (s *FetchStats) HumanSize() string

HumanSize returns the payload size in human-readable format.

type ID

ID is the source-owned spelling of the shared evidence source identity.

type ID = evidence.SourceID

func IDs
func IDs() []ID

IDs returns all available source identifiers.

type Observation

Observation is one immutable direct source result. EvidenceChecksum binds the normalized canonical catalog payload. Raw upstream evidence retention is a separate storage policy.

type Observation struct {
    ID               string                  `json:"id" yaml:"id"`
    SourceID         ID                      `json:"source" yaml:"source"`
    ObservedAt       time.Time               `json:"observed_at" yaml:"observed_at"`
    Revision         Revision                `json:"revision" yaml:"revision"`
    Completeness     ObservationCompleteness `json:"completeness" yaml:"completeness"`
    Status           ObservationStatus       `json:"status" yaml:"status"`
    Records          ObservationRecordCounts `json:"records" yaml:"records"`
    Issues           []ObservationIssue      `json:"issues,omitempty" yaml:"issues,omitempty"`
    EvidenceChecksum string                  `json:"evidence_checksum" yaml:"evidence_checksum"`
    Catalog          *catalogs.Catalog       `json:"-" yaml:"-"`
}

func NewObservation
func NewObservation(sourceID ID, catalog *catalogs.Catalog, metadata ObservationMetadata) (Observation, error)

NewObservation binds an immutable catalog to typed, deterministic audit metadata.

func (o Observation) Link() catalogs.SourceObservationLink

Link returns the immutable manifest/audit projection of this observation.

func (Observation) Validate
func (o Observation) Validate() error

Validate verifies required metadata and binds the evidence checksum to Catalog.

type ObservationCompleteness

ObservationCompleteness states whether an observation contains every expected record.

type ObservationCompleteness = evidence.ObservationCompleteness

type ObservationIssue

ObservationIssue records one classified, non-fatal degradation.

type ObservationIssue = evidence.ObservationIssue

type ObservationIssueCode

ObservationIssueCode is a stable machine-readable degradation reason.

type ObservationIssueCode = evidence.ObservationIssueCode

type ObservationIssueScope

ObservationIssueScope identifies the level at which degradation occurred.

type ObservationIssueScope = evidence.ObservationIssueScope

type ObservationMetadata

ObservationMetadata supplies source-owned metadata used to construct an observation.

type ObservationMetadata struct {
    ObservedAt   time.Time
    Revision     Revision
    Completeness ObservationCompleteness
    Status       ObservationStatus
    Records      ObservationRecordCounts
    Issues       []ObservationIssue
}

type ObservationRecordCounts

ObservationRecordCounts reports accepted and rejected source records.

type ObservationRecordCounts = evidence.ObservationRecordCounts

type ObservationStatus

ObservationStatus is the typed outcome of a source observation.

type ObservationStatus = evidence.ObservationStatus

type Option

Option is a function that configures options.

type Option func(*Options)

func WithCleanupRepo
func WithCleanupRepo(cleanup bool) Option

WithCleanupRepo configures whether to clean up temporary repositories after fetch.

func WithProviderFilter
func WithProviderFilter(providerID catalogs.ProviderID) Option

WithProviderFilter configures filtering for a specific provider.

func WithReformat
func WithReformat(reformat bool) Option

WithReformat configures whether to reformat output files.

type Options

Options is the configuration for sources.

type Options struct {
    // Provider filtering (needed by provider source)
    ProviderID *catalogs.ProviderID

    // Typed source-specific options
    CleanupRepo bool // For models.dev git source - remove repository after fetch
    Reformat    bool // For file-based sources - reformat output files
}

func Defaults
func Defaults() *Options

Defaults returns source options with default values.

func (*Options) Apply
func (o *Options) Apply(opts ...Option) *Options

Apply applies a set of options to create configured sourceOptions This is a helper for sources to use internally.

type ProviderAttempt

ProviderAttempt records one terminal provider acquisition attempt. It holds only values that are safe to log, to serve, and to retain.

type ProviderAttempt struct {
    // ProviderID names the attempted provider.
    ProviderID catalogs.ProviderID

    // Outcome is the terminal state of the attempt.
    Outcome ProviderOutcome

    // Reason explains a skip or a failure. It is empty for a success.
    Reason ProviderReason

    // Requested reports whether the attempt sent a provider request. A skip
    // for a missing credential never sends one.
    Requested bool

    // StartedAt and CompletedAt bound the attempt.
    StartedAt   time.Time
    CompletedAt time.Time

    // Records is the number of accepted provider records.
    Records int
}

func (ProviderAttempt) Validate
func (a ProviderAttempt) Validate() error

Validate checks that the attempt carries a defined outcome and a defined reason for every state other than success.

type ProviderClient

ProviderClient fetches model information from a provider API.

type ProviderClient interface {
    ListModels(ctx context.Context, material ProviderCredentialMaterial) ([]catalogs.Model, error)
}

type ProviderClientFactory

ProviderClientFactory creates provider API clients.

type ProviderClientFactory func(*catalogs.Provider) (ProviderClient, error)

type ProviderCredentialLease

ProviderCredentialLease describes renewable credential material.

type ProviderCredentialLease struct {
    Renewable    bool
    RefreshAfter time.Time
}

type ProviderCredentialMaterial

ProviderCredentialMaterial carries one selected catalog-acquisition profile and its resolved values. Values are private so generic serializers and formatters cannot expose them.

type ProviderCredentialMaterial struct {
    // contains filtered or unexported fields
}

func NewProviderCredentialMaterial
func NewProviderCredentialMaterial(profile catalogs.ProviderCredentialProfile, values map[catalogs.ProviderCredentialFieldID]string, metadata ProviderCredentialMetadata) ProviderCredentialMaterial

NewProviderCredentialMaterial creates caller-owned credential material.

func (ProviderCredentialMaterial) EndpointBindings
func (m ProviderCredentialMaterial) EndpointBindings() map[string]string

EndpointBindings returns resolved URL-template bindings for the profile.

func (ProviderCredentialMaterial) ExpiresAt
func (m ProviderCredentialMaterial) ExpiresAt() (time.Time, bool)

ExpiresAt returns the material expiry when the selected source supplied one.

func (ProviderCredentialMaterial) GoString
func (m ProviderCredentialMaterial) GoString() string

GoString returns a secret-free Go-syntax material summary.

func (ProviderCredentialMaterial) Lease
func (m ProviderCredentialMaterial) Lease() (ProviderCredentialLease, bool)

Lease returns caller-owned renewable-material metadata when present.

func (ProviderCredentialMaterial) Profile
func (m ProviderCredentialMaterial) Profile() catalogs.ProviderCredentialProfile

Profile returns a caller-owned copy of the selected profile.

func (ProviderCredentialMaterial) String
func (m ProviderCredentialMaterial) String() string

String returns a secret-free material summary.

func (ProviderCredentialMaterial) Value
func (m ProviderCredentialMaterial) Value(fieldID catalogs.ProviderCredentialFieldID) (string, bool)

Value returns one exact credential or parameter value.

func (ProviderCredentialMaterial) Version
func (m ProviderCredentialMaterial) Version() string

Version returns the resolver-owned opaque material version.

type ProviderCredentialMetadata

ProviderCredentialMetadata describes one resolved material lifecycle. Version is opaque and contains no source path or secret digest.

type ProviderCredentialMetadata struct {
    Version   string
    ExpiresAt time.Time
    Lease     *ProviderCredentialLease
}

type ProviderCredentialResolver

ProviderCredentialResolver resolves one catalog-acquisition profile.

type ProviderCredentialResolver interface {
    ResolveCatalog(context.Context, *catalogs.Provider) (ProviderCredentialMaterial, error)
}

type ProviderCredentialResolverFunc

ProviderCredentialResolverFunc adapts a function to credential resolution.

type ProviderCredentialResolverFunc func(
    context.Context,
    *catalogs.Provider,
) (ProviderCredentialMaterial, error)

func (ProviderCredentialResolverFunc) ResolveCatalog
func (f ProviderCredentialResolverFunc) ResolveCatalog(ctx context.Context, provider *catalogs.Provider) (ProviderCredentialMaterial, error)

ResolveCatalog implements ProviderCredentialResolver.

type ProviderFetcher

ProviderFetcher provides operations for fetching models from provider APIs. Concrete provider clients are an explicit injected composition. Use package acquisition for Starmap's built-in provider implementations.

type ProviderFetcher struct {
    // contains filtered or unexported fields
}

func NewProviderFetcher
func NewProviderFetcher(providers catalogs.ProvidersReader, opts ...ProviderOption) *ProviderFetcher

NewProviderFetcher creates a provider fetcher over the supplied catalog providers. Callers must inject the provider-client and raw-fetch roles they use. The root library never selects concrete provider implementations.

func (*ProviderFetcher) FetchModels
func (pf *ProviderFetcher) FetchModels(ctx context.Context, provider *catalogs.Provider, opts ...ProviderOption) ([]catalogs.Model, error)

FetchModels fetches available models from a single provider's API. It handles credential resolution, client creation, and API communication. When a provider quarantines malformed records, FetchModels returns the valid siblings together with a non-nil *sourcepayload.QuarantineError wrapped in a SyncError. Callers may consume the partial result only as degraded evidence.

Example:

fetcher := NewProviderFetcher(providers, WithProviderClientFactory(factory))
models, err := fetcher.FetchModels(ctx, provider)

With options:

fetcher := NewProviderFetcher(providers,
    WithProviderClientFactory(factory),
    WithTimeout(30*time.Second),
)
models, err := fetcher.FetchModels(ctx, provider)

func (*ProviderFetcher) FetchRawResponse
func (pf *ProviderFetcher) FetchRawResponse(ctx context.Context, provider *catalogs.Provider, endpoint string, opts ...ProviderOption) ([]byte, *FetchStats, error)

FetchRawResponse fetches the raw API response from a provider's endpoint. This is useful for testing, debugging, or saving raw responses as testdata.

The endpoint parameter should be the full URL to the API endpoint. FetchRawResponse returns unparsed JSON bytes and fetch statistics.

func (*ProviderFetcher) HasClient
func (pf *ProviderFetcher) HasClient(id catalogs.ProviderID) bool

HasClient checks if a provider ID has a client implementation.

func (*ProviderFetcher) List
func (pf *ProviderFetcher) List() []catalogs.ProviderID

List returns all provider IDs that have client implementations.

func (*ProviderFetcher) Providers
func (pf *ProviderFetcher) Providers() *catalogs.Providers

Providers returns the providers available to the provider fetcher.

type ProviderOption

ProviderOption configures ProviderFetcher behavior.

type ProviderOption func(*providerOptions)

func WithProviderClientFactory
func WithProviderClientFactory(factory ProviderClientFactory) ProviderOption

WithProviderClientFactory configures the factory used to create provider API clients.

func WithProviderCredentialResolver
func WithProviderCredentialResolver(resolver ProviderCredentialResolver) ProviderOption

WithProviderCredentialResolver configures catalog credential resolution.

func WithProviderRawFetcher
func WithProviderRawFetcher(fetcher ProviderRawFetcher) ProviderOption

WithProviderRawFetcher configures the raw provider response fetcher.

func WithTimeout
func WithTimeout(d time.Duration) ProviderOption

WithTimeout sets a timeout for provider operations. The timeout applies to the context passed to FetchModels.

type ProviderOutcome

ProviderOutcome is the terminal state of one provider acquisition attempt. Every attempt reaches exactly one outcome.

type ProviderOutcome string

const (
    // ProviderOutcomeSucceeded means the provider answered and the observation
    // carries provider records.
    ProviderOutcomeSucceeded ProviderOutcome = "succeeded"

    // ProviderOutcomeSkippedNotConfigured means the deployment holds no
    // catalog-acquisition credential for the provider. Acquisition sends no
    // request for a provider with this outcome.
    ProviderOutcomeSkippedNotConfigured ProviderOutcome = "skipped_not_configured"

    // ProviderOutcomeFailed means the attempt reached the provider and did not
    // produce a usable observation.
    ProviderOutcomeFailed ProviderOutcome = "failed"
)

func (ProviderOutcome) String
func (o ProviderOutcome) String() string

String returns the wire value of the outcome.

func (ProviderOutcome) Valid
func (o ProviderOutcome) Valid() bool

Valid reports whether the outcome is one of the three defined states.

type ProviderRawFetcher

ProviderRawFetcher fetches a raw provider API response.

type ProviderRawFetcher func(
    context.Context,
    *catalogs.Provider,
    ProviderCredentialMaterial,
    string,
) (*RawFetchResult, error)

type ProviderReason

ProviderReason is a safe machine-readable cause for a skip or a failure. A reason names no URL, no host, no token, and no credential value.

type ProviderReason string

const (
    // ProviderReasonCredentialReferenceInvalid means the credential reference
    // does not parse or names an unsupported source.
    ProviderReasonCredentialReferenceInvalid ProviderReason = "credential_reference_invalid" //nolint:gosec // A reason code holds no credential value.

    // ProviderReasonCredentialUnavailable means the deployment holds no credential.
    ProviderReasonCredentialUnavailable ProviderReason = "credential_unavailable" //nolint:gosec // A reason code holds no credential value.

    // ProviderReasonCredentialRejected means the provider refused the credential.
    ProviderReasonCredentialRejected ProviderReason = "credential_rejected" //nolint:gosec // A reason code holds no credential value.

    // ProviderReasonCredentialExpired means the credential is past its expiry.
    ProviderReasonCredentialExpired ProviderReason = "credential_expired" //nolint:gosec // A reason code holds no credential value.

    // ProviderReasonInsufficientScope means the credential lacks a required scope.
    ProviderReasonInsufficientScope ProviderReason = "insufficient_scope"

    // ProviderReasonRateLimited means the provider applied a request budget.
    ProviderReasonRateLimited ProviderReason = "rate_limited"

    // ProviderReasonRequestTimeout means the attempt exceeded its time budget.
    ProviderReasonRequestTimeout ProviderReason = "request_timeout"

    // ProviderReasonTransportFailed means the connection failed before a reply.
    ProviderReasonTransportFailed ProviderReason = "transport_failed"

    // ProviderReasonResponseInvalid means the reply did not match the contract.
    ProviderReasonResponseInvalid ProviderReason = "response_invalid"
)

func ClassifyProviderReason
func ClassifyProviderReason(err error) ProviderReason

ClassifyProviderReason maps one acquisition error onto a safe reason code. It reads typed error structure first, so provider message text reaches the result only through the final transport classification.

func ProviderReasons
func ProviderReasons() []ProviderReason

ProviderReasons returns a caller-owned copy of every defined reason code.

func (ProviderReason) String
func (r ProviderReason) String() string

String returns the wire value of the reason.

func (ProviderReason) Valid
func (r ProviderReason) Valid() bool

Valid reports whether the reason is one of the defined safe reason codes.

type RawFetchResult

RawFetchResult contains the result of a raw provider fetch operation.

type RawFetchResult struct {
    Data       []byte
    Response   *http.Response
    Latency    time.Duration
    RequestURL string
}

type Revision

Revision identifies the exact upstream or normalized content revision.

type Revision = evidence.ObservationRevision

type RevisionKind

RevisionKind identifies the source of an upstream observation revision.

type RevisionKind = evidence.ObservationRevisionKind

type SchemaDriftDisposition

SchemaDriftDisposition defines the response to a mismatch or unknown member.

type SchemaDriftDisposition string

const (
    // SchemaDriftRejectSource rejects a structurally unusable source observation.
    SchemaDriftRejectSource SchemaDriftDisposition = "reject_source"
    // SchemaDriftRejectRecord quarantines one malformed record and preserves valid siblings.
    SchemaDriftRejectRecord SchemaDriftDisposition = "reject_record"
    // SchemaDriftClassify preserves a fingerprint/evidence record for review without promotion.
    SchemaDriftClassify SchemaDriftDisposition = "classify"
    // SchemaDriftPreserve retains the exact value inside the source extension boundary.
    SchemaDriftPreserve SchemaDriftDisposition = "preserve"
    // SchemaDriftNotApplicable means the disposition does not apply at this path.
    SchemaDriftNotApplicable SchemaDriftDisposition = "n/a"
)

type SchemaDriftPolicy

SchemaDriftPolicy is the executable strict/tolerant contract for one path.

type SchemaDriftPolicy struct {
    Record       SchemaRecord
    Path         string
    Class        SchemaFieldClass
    Required     bool
    Mismatch     SchemaDriftDisposition
    UnknownField SchemaDriftDisposition
    Rationale    string
}

func SchemaDriftPolicies
func SchemaDriftPolicies(record SchemaRecord) []SchemaDriftPolicy

SchemaDriftPolicies returns caller-owned policies for a source record shape.

type SchemaFieldClass

SchemaFieldClass explains why a field boundary is strict or tolerant.

type SchemaFieldClass string

const (
    // SchemaFieldIdentity is an identity field whose drift rejects its scope.
    SchemaFieldIdentity SchemaFieldClass = "strict_identity"
    // SchemaFieldContainer is an object/array boundary whose type drift rejects its scope.
    SchemaFieldContainer SchemaFieldClass = "strict_container"
    // SchemaFieldValue is a known scalar value validated before canonical promotion.
    SchemaFieldValue SchemaFieldClass = "validated_value"
    // SchemaFieldExtension is an explicitly lossless source-extension boundary.
    SchemaFieldExtension SchemaFieldClass = "tolerant_extension"
)

type SchemaRecord

SchemaRecord identifies one independently validated source record shape.

type SchemaRecord string

const (
    // SchemaRecordObservation is the source observation envelope.
    SchemaRecordObservation SchemaRecord = "observation"
    // SchemaRecordCatalog is a complete decoded source catalog.
    SchemaRecordCatalog SchemaRecord = "catalog"
    // SchemaRecordProvider is one provider record inside a source catalog.
    SchemaRecordProvider SchemaRecord = "provider"
    // SchemaRecordModel is one source model record before canonical promotion.
    SchemaRecordModel SchemaRecord = "model"
    // SchemaRecordModelDefinition is one canonical provider-independent definition.
    SchemaRecordModelDefinition SchemaRecord = "model_definition"
    // SchemaRecordProviderOffering is one canonical provider-scoped offering.
    SchemaRecordProviderOffering SchemaRecord = "provider_offering"
)

type Source

Source observes catalog information from one configured upstream.

Implementations must be safe for repeated and concurrent Observe calls. Observe returns the complete result of that call directly and must not require a prior call or publish mutable result state through the Source.

type Source interface {
    // ID returns the stable identity of this source.
    ID() ID

    // Observe retrieves and returns one immutable source result directly. Calls
    // must not depend on prior Observe calls or publish result state on Source.
    Observe(ctx context.Context, opts ...Option) (Observation, error)

    Cleanup() error

    // Dependencies returns the list of external dependencies this source requires
    Dependencies() []Dependency

    // IsOptional returns true if the sync can succeed without this source
    IsOptional() bool
}

Generated by gomarkdoc

Documentation

Overview

Package sources provides public APIs for working with AI model data sources.

Index

Constants

View Source
const (
	// RevisionKindUnknown means the upstream exposes no stable revision.
	RevisionKindUnknown = evidence.ObservationRevisionKindUnknown
	// RevisionKindETag identifies an HTTP entity-tag revision.
	RevisionKindETag = evidence.ObservationRevisionKindETag
	// RevisionKindLastModified identifies an HTTP Last-Modified validator.
	RevisionKindLastModified = evidence.ObservationRevisionKindLastModified
	// RevisionKindGitCommit identifies an exact Git commit.
	RevisionKindGitCommit = evidence.ObservationRevisionKindGitCommit
	// RevisionKindSourceVersion identifies an upstream-declared version.
	RevisionKindSourceVersion = evidence.ObservationRevisionKindSourceVersion
	// RevisionKindContentDigest identifies the normalized observation content.
	RevisionKindContentDigest = evidence.ObservationRevisionKindContentDigest
)
View Source
const (
	// ObservationCompletenessComplete means the observation contains every expected record.
	ObservationCompletenessComplete = evidence.ObservationCompletenessComplete
	// ObservationCompletenessPartial means at least one expected record is absent.
	ObservationCompletenessPartial = evidence.ObservationCompletenessPartial
)
View Source
const (
	// ObservationStatusSucceeded means the observation completed without known degradation.
	ObservationStatusSucceeded = evidence.ObservationStatusSucceeded
	// ObservationStatusDegraded means the observation contains known issues.
	ObservationStatusDegraded = evidence.ObservationStatusDegraded
)
View Source
const (
	ObservationIssueScopeRecord        = evidence.ObservationIssueScopeRecord
	ObservationIssueScopeProvider      = evidence.ObservationIssueScopeProvider
	ObservationIssueScopeSource        = evidence.ObservationIssueScopeSource
	ObservationIssueScopeStaleFallback = evidence.ObservationIssueScopeStaleFallback
)

Observation issue scope values.

View Source
const (
	ObservationIssueCodeInvalidRecord      = evidence.ObservationIssueCodeInvalidRecord
	ObservationIssueCodeSchemaDrift        = evidence.ObservationIssueCodeSchemaDrift
	ObservationIssueCodePayloadLimit       = evidence.ObservationIssueCodePayloadLimit
	ObservationIssueCodeMissingCredentials = evidence.ObservationIssueCodeMissingCredentials
	ObservationIssueCodeConfiguration      = evidence.ObservationIssueCodeConfiguration
	ObservationIssueCodeFetchFailed        = evidence.ObservationIssueCodeFetchFailed
	ObservationIssueCodeStaleFallback      = evidence.ObservationIssueCodeStaleFallback
	ObservationIssueCodeBootstrapFallback  = evidence.ObservationIssueCodeBootstrapFallback
	ObservationIssueCodeVolumeCollapse     = evidence.ObservationIssueCodeVolumeCollapse
)

Observation issue code values.

View Source
const (
	ProvidersID       = evidence.ProvidersID
	ModelsDevGitID    = evidence.ModelsDevGitID
	ModelsDevHTTPID   = evidence.ModelsDevHTTPID
	LocalCatalogID    = evidence.LocalCatalogID
	ReleaseArtifactID = evidence.ReleaseArtifactID
	EmbeddedCatalogID = evidence.EmbeddedCatalogID
)

Source identifiers use names that read clearly at the Source API.

View Source
const MaxJSONNestingDepth = sourcepayload.MaxJSONNestingDepth

MaxJSONNestingDepth bounds object/array nesting before JSON decode.

Variables

This section is empty.

Functions

func ValidateJSONPayload added in v0.1.0

func ValidateJSONPayload(data []byte) error

ValidateJSONPayload enforces source byte and nesting limits before decoding.

Types

type Dependency added in v0.0.17

type Dependency struct {
	// Core identification
	Name        string // Machine name: "bun", "git", "docker"
	DisplayName string // Human-readable: "Bun JavaScript runtime"
	Required    bool   // false = source is optional or has fallback

	// Checking availability
	CheckCommands []string // Try in order: ["bun", "bunx"]
	MinVersion    string   // Optional: "1.0.0"

	// Installation
	InstallURL         string // https://bun.sh/docs/installation
	AutoInstallCommand string // Optional: "curl -fsSL https://bun.sh/install | bash"

	// User messaging
	Description       string // "Builds models.dev data locally (same as HTTP source)"
	WhyNeeded         string // "Required to build api.json from TypeScript source"
	AlternativeSource string // "models_dev_http provides same data without dependencies"
}

Dependency represents an external tool or runtime required by a source.

type DependencyStatus added in v0.0.17

type DependencyStatus struct {
	Available  bool   // Whether the dependency is available
	Version    string // Version string if available and detectable
	Path       string // Full path to executable if found
	CheckError error  // Error from check command if not available
}

DependencyStatus represents the availability status of a dependency.

type FetchStats added in v0.0.21

type FetchStats struct {
	URL          string
	StatusCode   int           // HTTP response status code
	Latency      time.Duration // Request duration
	PayloadSize  int64         // Response body size in bytes
	ContentType  string        // Content-Type from response header
	AuthMethod   string
	AuthLocation string
	AuthScheme   string // Authentication scheme for header auth (Bearer, Basic, Direct)
}

FetchStats contains metadata about a fetch operation. This provides transparency into API requests for debugging and monitoring.

func (*FetchStats) HumanSize added in v0.0.21

func (s *FetchStats) HumanSize() string

HumanSize returns the payload size in human-readable format.

type ID added in v0.0.15

type ID = evidence.SourceID

ID is the source-owned spelling of the shared evidence source identity.

func IDs added in v0.0.15

func IDs() []ID

IDs returns all available source identifiers.

type Observation added in v0.1.0

type Observation struct {
	ID               string                  `json:"id" yaml:"id"`
	SourceID         ID                      `json:"source" yaml:"source"`
	ObservedAt       time.Time               `json:"observed_at" yaml:"observed_at"`
	Revision         Revision                `json:"revision" yaml:"revision"`
	Completeness     ObservationCompleteness `json:"completeness" yaml:"completeness"`
	Status           ObservationStatus       `json:"status" yaml:"status"`
	Records          ObservationRecordCounts `json:"records" yaml:"records"`
	Issues           []ObservationIssue      `json:"issues,omitempty" yaml:"issues,omitempty"`
	EvidenceChecksum string                  `json:"evidence_checksum" yaml:"evidence_checksum"`
	Catalog          *catalogs.Catalog       `json:"-" yaml:"-"`
}

Observation is one immutable direct source result. EvidenceChecksum binds the normalized canonical catalog payload. Raw upstream evidence retention is a separate storage policy.

func NewObservation added in v0.1.0

func NewObservation(sourceID ID, catalog *catalogs.Catalog, metadata ObservationMetadata) (Observation, error)

NewObservation binds an immutable catalog to typed, deterministic audit metadata.

Link returns the immutable manifest/audit projection of this observation.

func (Observation) Validate added in v0.1.0

func (o Observation) Validate() error

Validate verifies required metadata and binds the evidence checksum to Catalog.

type ObservationCompleteness added in v0.1.0

type ObservationCompleteness = evidence.ObservationCompleteness

ObservationCompleteness states whether an observation contains every expected record.

type ObservationIssue added in v0.1.0

type ObservationIssue = evidence.ObservationIssue

ObservationIssue records one classified, non-fatal degradation.

type ObservationIssueCode added in v0.1.0

type ObservationIssueCode = evidence.ObservationIssueCode

ObservationIssueCode is a stable machine-readable degradation reason.

type ObservationIssueScope added in v0.1.0

type ObservationIssueScope = evidence.ObservationIssueScope

ObservationIssueScope identifies the level at which degradation occurred.

type ObservationMetadata added in v0.1.0

type ObservationMetadata struct {
	ObservedAt   time.Time
	Revision     Revision
	Completeness ObservationCompleteness
	Status       ObservationStatus
	Records      ObservationRecordCounts
	Issues       []ObservationIssue
}

ObservationMetadata supplies source-owned metadata used to construct an observation.

type ObservationRecordCounts added in v0.1.0

type ObservationRecordCounts = evidence.ObservationRecordCounts

ObservationRecordCounts reports accepted and rejected source records.

type ObservationStatus added in v0.1.0

type ObservationStatus = evidence.ObservationStatus

ObservationStatus is the typed outcome of a source observation.

type Option

type Option func(*Options)

Option is a function that configures options.

func WithCleanupRepo

func WithCleanupRepo(cleanup bool) Option

WithCleanupRepo configures whether to clean up temporary repositories after fetch.

func WithProviderFilter

func WithProviderFilter(providerID catalogs.ProviderID) Option

WithProviderFilter configures filtering for a specific provider.

func WithReformat

func WithReformat(reformat bool) Option

WithReformat configures whether to reformat output files.

type Options

type Options struct {
	// Provider filtering (needed by provider source)
	ProviderID *catalogs.ProviderID

	// Typed source-specific options
	CleanupRepo bool // For models.dev git source - remove repository after fetch
	Reformat    bool // For file-based sources - reformat output files
}

Options is the configuration for sources.

func Defaults added in v0.0.15

func Defaults() *Options

Defaults returns source options with default values.

func (*Options) Apply added in v0.0.15

func (o *Options) Apply(opts ...Option) *Options

Apply applies a set of options to create configured sourceOptions This is a helper for sources to use internally.

type ProviderAttempt added in v0.16.0

type ProviderAttempt struct {
	// ProviderID names the attempted provider.
	ProviderID catalogs.ProviderID

	// Outcome is the terminal state of the attempt.
	Outcome ProviderOutcome

	// Reason explains a skip or a failure. It is empty for a success.
	Reason ProviderReason

	// Requested reports whether the attempt sent a provider request. A skip
	// for a missing credential never sends one.
	Requested bool

	// StartedAt and CompletedAt bound the attempt.
	StartedAt   time.Time
	CompletedAt time.Time

	// Records is the number of accepted provider records.
	Records int
}

ProviderAttempt records one terminal provider acquisition attempt. It holds only values that are safe to log, to serve, and to retain.

func (ProviderAttempt) Validate added in v0.16.0

func (a ProviderAttempt) Validate() error

Validate checks that the attempt carries a defined outcome and a defined reason for every state other than success.

type ProviderClient added in v0.1.0

type ProviderClient interface {
	ListModels(ctx context.Context, material ProviderCredentialMaterial) ([]catalogs.Model, error)
}

ProviderClient fetches model information from a provider API.

type ProviderClientFactory added in v0.1.0

type ProviderClientFactory func(*catalogs.Provider) (ProviderClient, error)

ProviderClientFactory creates provider API clients.

type ProviderCredentialLease added in v0.4.0

type ProviderCredentialLease struct {
	Renewable    bool
	RefreshAfter time.Time
}

ProviderCredentialLease describes renewable credential material.

type ProviderCredentialMaterial added in v0.4.0

type ProviderCredentialMaterial struct {
	// contains filtered or unexported fields
}

ProviderCredentialMaterial carries one selected catalog-acquisition profile and its resolved values. Values are private so generic serializers and formatters cannot expose them.

func NewProviderCredentialMaterial added in v0.4.0

NewProviderCredentialMaterial creates caller-owned credential material.

func (ProviderCredentialMaterial) EndpointBindings added in v0.4.0

func (m ProviderCredentialMaterial) EndpointBindings() map[string]string

EndpointBindings returns resolved URL-template bindings for the profile.

func (ProviderCredentialMaterial) ExpiresAt added in v0.4.0

func (m ProviderCredentialMaterial) ExpiresAt() (time.Time, bool)

ExpiresAt returns the material expiry when the selected source supplied one.

func (ProviderCredentialMaterial) GoString added in v0.4.0

func (m ProviderCredentialMaterial) GoString() string

GoString returns a secret-free Go-syntax material summary.

func (ProviderCredentialMaterial) Lease added in v0.4.0

Lease returns caller-owned renewable-material metadata when present.

func (ProviderCredentialMaterial) Profile added in v0.4.0

Profile returns a caller-owned copy of the selected profile.

func (ProviderCredentialMaterial) String added in v0.4.0

String returns a secret-free material summary.

func (ProviderCredentialMaterial) Value added in v0.4.0

Value returns one exact credential or parameter value.

func (ProviderCredentialMaterial) Version added in v0.4.0

func (m ProviderCredentialMaterial) Version() string

Version returns the resolver-owned opaque material version.

type ProviderCredentialMetadata added in v0.4.0

type ProviderCredentialMetadata struct {
	Version   string
	ExpiresAt time.Time
	Lease     *ProviderCredentialLease
}

ProviderCredentialMetadata describes one resolved material lifecycle. Version is opaque and contains no source path or secret digest.

type ProviderCredentialResolver added in v0.4.0

type ProviderCredentialResolver interface {
	ResolveCatalog(context.Context, *catalogs.Provider) (ProviderCredentialMaterial, error)
}

ProviderCredentialResolver resolves one catalog-acquisition profile.

type ProviderCredentialResolverFunc added in v0.4.0

type ProviderCredentialResolverFunc func(
	context.Context,
	*catalogs.Provider,
) (ProviderCredentialMaterial, error)

ProviderCredentialResolverFunc adapts a function to credential resolution.

func (ProviderCredentialResolverFunc) ResolveCatalog added in v0.4.0

ResolveCatalog implements ProviderCredentialResolver.

type ProviderFetcher

type ProviderFetcher struct {
	// contains filtered or unexported fields
}

ProviderFetcher provides operations for fetching models from provider APIs. Concrete provider clients are an explicit injected composition. Use package acquisition for Starmap's built-in provider implementations.

func NewProviderFetcher

func NewProviderFetcher(providers catalogs.ProvidersReader, opts ...ProviderOption) *ProviderFetcher

NewProviderFetcher creates a provider fetcher over the supplied catalog providers. Callers must inject the provider-client and raw-fetch roles they use. The root library never selects concrete provider implementations.

func (*ProviderFetcher) FetchModels

func (pf *ProviderFetcher) FetchModels(ctx context.Context, provider *catalogs.Provider, opts ...ProviderOption) ([]catalogs.Model, error)

FetchModels fetches available models from a single provider's API. It handles credential resolution, client creation, and API communication. When a provider quarantines malformed records, FetchModels returns the valid siblings together with a non-nil *sourcepayload.QuarantineError wrapped in a SyncError. Callers may consume the partial result only as degraded evidence.

Example:

fetcher := NewProviderFetcher(providers, WithProviderClientFactory(factory))
models, err := fetcher.FetchModels(ctx, provider)

With options:

fetcher := NewProviderFetcher(providers,
    WithProviderClientFactory(factory),
    WithTimeout(30*time.Second),
)
models, err := fetcher.FetchModels(ctx, provider)

func (*ProviderFetcher) FetchRawResponse

func (pf *ProviderFetcher) FetchRawResponse(ctx context.Context, provider *catalogs.Provider, endpoint string, opts ...ProviderOption) ([]byte, *FetchStats, error)

FetchRawResponse fetches the raw API response from a provider's endpoint. This is useful for testing, debugging, or saving raw responses as testdata.

The endpoint parameter should be the full URL to the API endpoint. FetchRawResponse returns unparsed JSON bytes and fetch statistics.

func (*ProviderFetcher) HasClient

func (pf *ProviderFetcher) HasClient(id catalogs.ProviderID) bool

HasClient checks if a provider ID has a client implementation.

func (*ProviderFetcher) List

func (pf *ProviderFetcher) List() []catalogs.ProviderID

List returns all provider IDs that have client implementations.

func (*ProviderFetcher) Providers added in v0.0.15

func (pf *ProviderFetcher) Providers() *catalogs.Providers

Providers returns the providers available to the provider fetcher.

type ProviderOption

type ProviderOption func(*providerOptions)

ProviderOption configures ProviderFetcher behavior.

func WithProviderClientFactory added in v0.1.0

func WithProviderClientFactory(factory ProviderClientFactory) ProviderOption

WithProviderClientFactory configures the factory used to create provider API clients.

func WithProviderCredentialResolver added in v0.4.0

func WithProviderCredentialResolver(resolver ProviderCredentialResolver) ProviderOption

WithProviderCredentialResolver configures catalog credential resolution.

func WithProviderRawFetcher added in v0.1.0

func WithProviderRawFetcher(fetcher ProviderRawFetcher) ProviderOption

WithProviderRawFetcher configures the raw provider response fetcher.

func WithTimeout

func WithTimeout(d time.Duration) ProviderOption

WithTimeout sets a timeout for provider operations. The timeout applies to the context passed to FetchModels.

type ProviderOutcome added in v0.16.0

type ProviderOutcome string

ProviderOutcome is the terminal state of one provider acquisition attempt. Every attempt reaches exactly one outcome.

const (
	// ProviderOutcomeSucceeded means the provider answered and the observation
	// carries provider records.
	ProviderOutcomeSucceeded ProviderOutcome = "succeeded"

	// ProviderOutcomeSkippedNotConfigured means the deployment holds no
	// catalog-acquisition credential for the provider. Acquisition sends no
	// request for a provider with this outcome.
	ProviderOutcomeSkippedNotConfigured ProviderOutcome = "skipped_not_configured"

	// ProviderOutcomeFailed means the attempt reached the provider and did not
	// produce a usable observation.
	ProviderOutcomeFailed ProviderOutcome = "failed"
)

func (ProviderOutcome) String added in v0.16.0

func (o ProviderOutcome) String() string

String returns the wire value of the outcome.

func (ProviderOutcome) Valid added in v0.16.0

func (o ProviderOutcome) Valid() bool

Valid reports whether the outcome is one of the three defined states.

type ProviderRawFetcher added in v0.1.0

ProviderRawFetcher fetches a raw provider API response.

type ProviderReason added in v0.16.0

type ProviderReason string

ProviderReason is a safe machine-readable cause for a skip or a failure. A reason names no URL, no host, no token, and no credential value.

const (
	// ProviderReasonCredentialReferenceInvalid means the credential reference
	// does not parse or names an unsupported source.
	ProviderReasonCredentialReferenceInvalid ProviderReason = "credential_reference_invalid" //nolint:gosec // A reason code holds no credential value.

	// ProviderReasonCredentialUnavailable means the deployment holds no credential.
	ProviderReasonCredentialUnavailable ProviderReason = "credential_unavailable" //nolint:gosec // A reason code holds no credential value.

	// ProviderReasonCredentialRejected means the provider refused the credential.
	ProviderReasonCredentialRejected ProviderReason = "credential_rejected" //nolint:gosec // A reason code holds no credential value.

	// ProviderReasonCredentialExpired means the credential is past its expiry.
	ProviderReasonCredentialExpired ProviderReason = "credential_expired" //nolint:gosec // A reason code holds no credential value.

	// ProviderReasonInsufficientScope means the credential lacks a required scope.
	ProviderReasonInsufficientScope ProviderReason = "insufficient_scope"

	// ProviderReasonRateLimited means the provider applied a request budget.
	ProviderReasonRateLimited ProviderReason = "rate_limited"

	// ProviderReasonRequestTimeout means the attempt exceeded its time budget.
	ProviderReasonRequestTimeout ProviderReason = "request_timeout"

	// ProviderReasonTransportFailed means the connection failed before a reply.
	ProviderReasonTransportFailed ProviderReason = "transport_failed"

	// ProviderReasonResponseInvalid means the reply did not match the contract.
	ProviderReasonResponseInvalid ProviderReason = "response_invalid"
)

func ClassifyProviderReason added in v0.16.0

func ClassifyProviderReason(err error) ProviderReason

ClassifyProviderReason maps one acquisition error onto a safe reason code. It reads typed error structure first, so provider message text reaches the result only through the final transport classification.

func ProviderReasons added in v0.16.0

func ProviderReasons() []ProviderReason

ProviderReasons returns a caller-owned copy of every defined reason code.

func (ProviderReason) String added in v0.16.0

func (r ProviderReason) String() string

String returns the wire value of the reason.

func (ProviderReason) Valid added in v0.16.0

func (r ProviderReason) Valid() bool

Valid reports whether the reason is one of the defined safe reason codes.

type RawFetchResult added in v0.1.0

type RawFetchResult struct {
	Data       []byte
	Response   *http.Response
	Latency    time.Duration
	RequestURL string
}

RawFetchResult contains the result of a raw provider fetch operation.

type Revision added in v0.1.0

type Revision = evidence.ObservationRevision

Revision identifies the exact upstream or normalized content revision.

type RevisionKind added in v0.1.0

type RevisionKind = evidence.ObservationRevisionKind

RevisionKind identifies the source of an upstream observation revision.

type SchemaDriftDisposition added in v0.1.0

type SchemaDriftDisposition string

SchemaDriftDisposition defines the response to a mismatch or unknown member.

const (
	// SchemaDriftRejectSource rejects a structurally unusable source observation.
	SchemaDriftRejectSource SchemaDriftDisposition = "reject_source"
	// SchemaDriftRejectRecord quarantines one malformed record and preserves valid siblings.
	SchemaDriftRejectRecord SchemaDriftDisposition = "reject_record"
	// SchemaDriftClassify preserves a fingerprint/evidence record for review without promotion.
	SchemaDriftClassify SchemaDriftDisposition = "classify"
	// SchemaDriftPreserve retains the exact value inside the source extension boundary.
	SchemaDriftPreserve SchemaDriftDisposition = "preserve"
	// SchemaDriftNotApplicable means the disposition does not apply at this path.
	SchemaDriftNotApplicable SchemaDriftDisposition = "n/a"
)

type SchemaDriftPolicy added in v0.1.0

type SchemaDriftPolicy struct {
	Record       SchemaRecord
	Path         string
	Class        SchemaFieldClass
	Required     bool
	Mismatch     SchemaDriftDisposition
	UnknownField SchemaDriftDisposition
	Rationale    string
}

SchemaDriftPolicy is the executable strict/tolerant contract for one path.

func SchemaDriftPolicies added in v0.1.0

func SchemaDriftPolicies(record SchemaRecord) []SchemaDriftPolicy

SchemaDriftPolicies returns caller-owned policies for a source record shape.

type SchemaFieldClass added in v0.1.0

type SchemaFieldClass string

SchemaFieldClass explains why a field boundary is strict or tolerant.

const (
	// SchemaFieldIdentity is an identity field whose drift rejects its scope.
	SchemaFieldIdentity SchemaFieldClass = "strict_identity"
	// SchemaFieldContainer is an object/array boundary whose type drift rejects its scope.
	SchemaFieldContainer SchemaFieldClass = "strict_container"
	// SchemaFieldValue is a known scalar value validated before canonical promotion.
	SchemaFieldValue SchemaFieldClass = "validated_value"
	// SchemaFieldExtension is an explicitly lossless source-extension boundary.
	SchemaFieldExtension SchemaFieldClass = "tolerant_extension"
)

type SchemaRecord added in v0.1.0

type SchemaRecord string

SchemaRecord identifies one independently validated source record shape.

const (
	// SchemaRecordObservation is the source observation envelope.
	SchemaRecordObservation SchemaRecord = "observation"
	// SchemaRecordCatalog is a complete decoded source catalog.
	SchemaRecordCatalog SchemaRecord = "catalog"
	// SchemaRecordProvider is one provider record inside a source catalog.
	SchemaRecordProvider SchemaRecord = "provider"
	// SchemaRecordModel is one source model record before canonical promotion.
	SchemaRecordModel SchemaRecord = "model"
	// SchemaRecordModelDefinition is one canonical provider-independent definition.
	SchemaRecordModelDefinition SchemaRecord = "model_definition"
	// SchemaRecordProviderOffering is one canonical provider-scoped offering.
	SchemaRecordProviderOffering SchemaRecord = "provider_offering"
)

type Source

type Source interface {
	// ID returns the stable identity of this source.
	ID() ID

	// Observe retrieves and returns one immutable source result directly. Calls
	// must not depend on prior Observe calls or publish result state on Source.
	Observe(ctx context.Context, opts ...Option) (Observation, error)

	Cleanup() error

	// Dependencies returns the list of external dependencies this source requires
	Dependencies() []Dependency

	// IsOptional returns true if the sync can succeed without this source
	IsOptional() bool
}

Source observes catalog information from one configured upstream.

Implementations must be safe for repeated and concurrent Observe calls. Observe returns the complete result of that call directly and must not require a prior call or publish mutable result state through the Source.

Directories

Path Synopsis
Package payload enforces bounded resource use before source decoding.
Package payload enforces bounded resource use before source decoding.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL