sources

sources

Package sources provides abstractions for fetching AI model catalog data from various external sources including provider APIs and community repositories.

sources

import "github.com/agentstation/starmap/pkg/sources"

Package sources provides public APIs for working with AI model data sources.

Package sources defines interfaces and types for catalog data sources. Sources are responsible for fetching and synchronizing model data from various providers including local files, provider APIs, and external repositories.

The package provides a unified interface for different data sources while supporting merge strategies, authorities for data precedence, and flexible configuration options.

Example usage:

// Create a provider fetcher
fetcher := NewProviderFetcher(providers)

// Fetch models from a provider
models, err := fetcher.FetchModels(ctx, provider)
if err != nil {
    log.Fatal(err)
}

// Check if a provider is supported
if fetcher.HasClient(providerID) {
    // Provider has a client implementation
}

Index

• Constants
• func ValidateJSONPayload(data []byte) error
• type Dependency
• type DependencyStatus
• type FetchStats
  • func (s *FetchStats) HumanSize() string
• type ID
  • func IDs() []ID
• type Observation
  • func NewObservation(sourceID ID, catalog *catalogs.Catalog, metadata ObservationMetadata) (Observation, error)
  • func (o Observation) Link() catalogs.SourceObservationLink
  • func (o Observation) Validate() error
• type ObservationCompleteness
• type ObservationIssue
• type ObservationIssueCode
• type ObservationIssueScope
• type ObservationMetadata
• type ObservationRecordCounts
• type ObservationStatus
• type Option
  • func WithCleanupRepo(cleanup bool) Option
  • func WithProviderFilter(providerID catalogs.ProviderID) Option
  • func WithReformat(reformat bool) Option
• type Options
  • func Defaults() *Options
  • func (o *Options) Apply(opts ...Option) *Options
• type ProviderClient
• type ProviderClientFactory
• type ProviderFetcher
  • func NewProviderFetcher(providers catalogs.ProvidersReader, opts ...ProviderOption) *ProviderFetcher
  • func (pf *ProviderFetcher) FetchModels(ctx context.Context, provider *catalogs.Provider, opts ...ProviderOption) ([]catalogs.Model, error)
  • func (pf *ProviderFetcher) FetchRawResponse(ctx context.Context, provider *catalogs.Provider, endpoint string, opts ...ProviderOption) ([]byte, *FetchStats, error)
  • func (pf *ProviderFetcher) HasClient(id catalogs.ProviderID) bool
  • func (pf *ProviderFetcher) List() []catalogs.ProviderID
  • func (pf *ProviderFetcher) Providers() *catalogs.Providers
• type ProviderOption
  • func WithAllowMissingAPIKey() ProviderOption
  • func WithProviderClientFactory(factory ProviderClientFactory) ProviderOption
  • func WithProviderRawFetcher(fetcher ProviderRawFetcher) ProviderOption
  • func WithTimeout(d time.Duration) ProviderOption
  • func WithoutCredentialLoading() ProviderOption
• type ProviderRawFetcher
• type RawFetchResult
• type ResourceType
• type Revision
• type RevisionKind
• type SchemaDriftDisposition
• type SchemaDriftPolicy
  • func SchemaDriftPolicies(record SchemaRecord) []SchemaDriftPolicy
• type SchemaFieldClass
• type SchemaRecord
• type Source

Constants

const (
    // RevisionKindUnknown means the upstream exposes no stable revision.
    RevisionKindUnknown = catalogmeta.ObservationRevisionKindUnknown
    // RevisionKindETag identifies an HTTP entity-tag revision.
    RevisionKindETag = catalogmeta.ObservationRevisionKindETag
    // RevisionKindLastModified identifies an HTTP Last-Modified validator.
    RevisionKindLastModified = catalogmeta.ObservationRevisionKindLastModified
    // RevisionKindGitCommit identifies an exact Git commit.
    RevisionKindGitCommit = catalogmeta.ObservationRevisionKindGitCommit
    // RevisionKindSourceVersion identifies an upstream-declared version.
    RevisionKindSourceVersion = catalogmeta.ObservationRevisionKindSourceVersion
    // RevisionKindContentDigest identifies the normalized observation content.
    RevisionKindContentDigest = catalogmeta.ObservationRevisionKindContentDigest
)

const (
    // ObservationCompletenessComplete means every expected record was observed.
    ObservationCompletenessComplete = catalogmeta.ObservationCompletenessComplete
    // ObservationCompletenessPartial means at least one expected record is absent.
    ObservationCompletenessPartial = catalogmeta.ObservationCompletenessPartial
)

const (
    // ObservationStatusSucceeded means the observation completed without known degradation.
    ObservationStatusSucceeded = catalogmeta.ObservationStatusSucceeded
    // ObservationStatusDegraded means usable catalog data was returned with a known limitation.
    ObservationStatusDegraded = catalogmeta.ObservationStatusDegraded
)

Observation issue scope values.

const (
    ObservationIssueScopeRecord        = catalogmeta.ObservationIssueScopeRecord
    ObservationIssueScopeProvider      = catalogmeta.ObservationIssueScopeProvider
    ObservationIssueScopeSource        = catalogmeta.ObservationIssueScopeSource
    ObservationIssueScopeStaleFallback = catalogmeta.ObservationIssueScopeStaleFallback
)

Observation issue code values.

const (
    ObservationIssueCodeInvalidRecord      = catalogmeta.ObservationIssueCodeInvalidRecord
    ObservationIssueCodeSchemaDrift        = catalogmeta.ObservationIssueCodeSchemaDrift
    ObservationIssueCodePayloadLimit       = catalogmeta.ObservationIssueCodePayloadLimit
    ObservationIssueCodeMissingCredentials = catalogmeta.ObservationIssueCodeMissingCredentials
    ObservationIssueCodeConfiguration      = catalogmeta.ObservationIssueCodeConfiguration
    ObservationIssueCodeFetchFailed        = catalogmeta.ObservationIssueCodeFetchFailed
    ObservationIssueCodeStaleFallback      = catalogmeta.ObservationIssueCodeStaleFallback
    ObservationIssueCodeBootstrapFallback  = catalogmeta.ObservationIssueCodeBootstrapFallback
    ObservationIssueCodeVolumeCollapse     = catalogmeta.ObservationIssueCodeVolumeCollapse
)

Common source identifiers - exported as package-level constants for convenience.

const (
    ProvidersID       = catalogmeta.ProvidersID
    ModelsDevGitID    = catalogmeta.ModelsDevGitID
    ModelsDevHTTPID   = catalogmeta.ModelsDevHTTPID
    LocalCatalogID    = catalogmeta.LocalCatalogID
    ReleaseArtifactID = catalogmeta.ReleaseArtifactID
    EmbeddedCatalogID = catalogmeta.EmbeddedCatalogID
)

Common resource type identifiers - exported as package-level constants for convenience.

const (
    ResourceTypeModel            = catalogmeta.ResourceTypeModel
    ResourceTypeProvider         = catalogmeta.ResourceTypeProvider
    ResourceTypeAuthor           = catalogmeta.ResourceTypeAuthor
    ResourceTypeModelDefinition  = catalogmeta.ResourceTypeModelDefinition
    ResourceTypeProviderOffering = catalogmeta.ResourceTypeProviderOffering
)

MaxJSONNestingDepth bounds object/array nesting before JSON decode.

const MaxJSONNestingDepth = sourcepayload.MaxJSONNestingDepth

func ValidateJSONPayload

func ValidateJSONPayload(data []byte) error

ValidateJSONPayload enforces source byte and nesting limits before decoding.

type Dependency

Dependency represents an external tool or runtime required by a source.

type Dependency struct {
    // Core identification
    Name        string // Machine name: "bun", "git", "docker"
    DisplayName string // Human-readable: "Bun JavaScript runtime"
    Required    bool   // false = source is optional or has fallback

    // Checking availability
    CheckCommands []string // Try in order: ["bun", "bunx"]
    MinVersion    string   // Optional: "1.0.0"

    // Installation
    InstallURL         string // https://bun.sh/docs/installation
    AutoInstallCommand string // Optional: "curl -fsSL https://bun.sh/install | bash"

    // User messaging
    Description       string // "Builds models.dev data locally (same as HTTP source)"
    WhyNeeded         string // "Required to build api.json from TypeScript source"
    AlternativeSource string // "models_dev_http provides same data without dependencies"
}

type DependencyStatus

DependencyStatus represents the availability status of a dependency.

type DependencyStatus struct {
    Available  bool   // Whether the dependency is available
    Version    string // Version string if available and detectable
    Path       string // Full path to executable if found
    CheckError error  // Error from check command if not available
}

type FetchStats

FetchStats contains metadata about a fetch operation. This provides transparency into API requests for debugging and monitoring.

type FetchStats struct {
    URL          string        // Endpoint that was called
    StatusCode   int           // HTTP response status code
    Latency      time.Duration // Request duration
    PayloadSize  int64         // Response body size in bytes
    ContentType  string        // Content-Type from response header
    AuthMethod   string        // How authentication was applied (Header, Query, None)
    AuthLocation string        // Where auth was placed (header name or query param name)
    AuthScheme   string        // Authentication scheme for header auth (Bearer, Basic, Direct)
}

func (*FetchStats) HumanSize

func (s *FetchStats) HumanSize() string

HumanSize returns the payload size in human-readable format.

type ID

ID represents the identifier of a data source. ID is a type alias for catalogmeta.SourceID to maintain backward compatibility. This allows existing code to continue using sources.ID while benefiting from the shared type definitions in pkg/catalogmeta.

type ID = catalogmeta.SourceID

func IDs

func IDs() []ID

IDs returns all available source identifiers. Delegates to catalogmeta.SourceIDs() to maintain consistency.

type Observation

Observation is one immutable direct source result. EvidenceChecksum binds the normalized canonical catalog payload; raw upstream evidence retention is a separate storage policy.

type Observation struct {
    ID               string                  `json:"id" yaml:"id"`
    SourceID         ID                      `json:"source" yaml:"source"`
    ObservedAt       time.Time               `json:"observed_at" yaml:"observed_at"`
    Revision         Revision                `json:"revision" yaml:"revision"`
    Completeness     ObservationCompleteness `json:"completeness" yaml:"completeness"`
    Status           ObservationStatus       `json:"status" yaml:"status"`
    Records          ObservationRecordCounts `json:"records" yaml:"records"`
    Issues           []ObservationIssue      `json:"issues,omitempty" yaml:"issues,omitempty"`
    EvidenceChecksum string                  `json:"evidence_checksum" yaml:"evidence_checksum"`
    Catalog          *catalogs.Catalog       `json:"-" yaml:"-"`
}

func NewObservation

func NewObservation(sourceID ID, catalog *catalogs.Catalog, metadata ObservationMetadata) (Observation, error)

NewObservation binds an immutable catalog to typed, deterministic audit metadata.

func (Observation) Link

func (o Observation) Link() catalogs.SourceObservationLink

Link returns the immutable manifest/audit projection of this observation.

func (Observation) Validate

func (o Observation) Validate() error

Validate verifies required metadata and binds the evidence checksum to Catalog.

type ObservationCompleteness

ObservationCompleteness states whether all expected records were observed.

type ObservationCompleteness = catalogmeta.ObservationCompleteness

type ObservationIssue

ObservationIssue records one classified, non-fatal degradation.

type ObservationIssue = catalogmeta.ObservationIssue

type ObservationIssueCode

ObservationIssueCode is a stable machine-readable degradation reason.

type ObservationIssueCode = catalogmeta.ObservationIssueCode

type ObservationIssueScope

ObservationIssueScope identifies the level at which degradation occurred.

type ObservationIssueScope = catalogmeta.ObservationIssueScope

type ObservationMetadata

ObservationMetadata supplies source-owned metadata used to construct an observation.

type ObservationMetadata struct {
    ObservedAt   time.Time
    Revision     Revision
    Completeness ObservationCompleteness
    Status       ObservationStatus
    Records      ObservationRecordCounts
    Issues       []ObservationIssue
}

type ObservationRecordCounts

ObservationRecordCounts reports accepted and rejected source records.

type ObservationRecordCounts = catalogmeta.ObservationRecordCounts

type ObservationStatus

ObservationStatus is the typed outcome of a source observation.

type ObservationStatus = catalogmeta.ObservationStatus

type Option

Option is a function that configures options.

type Option func(*Options)

func WithCleanupRepo

func WithCleanupRepo(cleanup bool) Option

WithCleanupRepo configures whether to clean up temporary repositories after fetch.

func WithProviderFilter

func WithProviderFilter(providerID catalogs.ProviderID) Option

WithProviderFilter configures filtering for a specific provider.

func WithReformat

func WithReformat(reformat bool) Option

WithReformat configures whether to reformat output files.

type Options

Options is the configuration for sources.

type Options struct {
    // Provider filtering (needed by provider source)
    ProviderID *catalogs.ProviderID

    // Typed source-specific options
    CleanupRepo bool // For models.dev git source - remove repo after fetch
    Reformat    bool // For file-based sources - reformat output files
}

func Defaults

func Defaults() *Options

Defaults returns source options with default values.

func (*Options) Apply

func (o *Options) Apply(opts ...Option) *Options

Apply applies a set of options to create configured sourceOptions This is a helper for sources to use internally.

type ProviderClient

ProviderClient fetches model information from a provider API.

type ProviderClient interface {
    ListModels(ctx context.Context) ([]catalogs.Model, error)
    IsAPIKeyRequired() bool
    HasAPIKey() bool
}

type ProviderClientFactory

ProviderClientFactory creates provider API clients.

type ProviderClientFactory func(*catalogs.Provider) (ProviderClient, error)

type ProviderFetcher

ProviderFetcher provides operations for fetching models from provider APIs. Concrete provider clients are an explicit injected composition; use package acquisition for Starmap's built-in provider implementations.

type ProviderFetcher struct {
    // contains filtered or unexported fields
}

func NewProviderFetcher

func NewProviderFetcher(providers catalogs.ProvidersReader, opts ...ProviderOption) *ProviderFetcher

NewProviderFetcher creates a provider fetcher over the supplied catalog providers. Callers must inject the provider-client and raw-fetch roles they use; the root library never selects concrete provider implementations.

func (*ProviderFetcher) FetchModels

func (pf *ProviderFetcher) FetchModels(ctx context.Context, provider *catalogs.Provider, opts ...ProviderOption) ([]catalogs.Model, error)

FetchModels fetches available models from a single provider's API. It handles credential loading, client creation, and API communication. When a provider quarantines malformed records, FetchModels returns the valid siblings together with a non-nil *sourcepayload.QuarantineError wrapped in a SyncError; callers may consume the partial result only as degraded evidence.

Example:

fetcher := NewProviderFetcher(providers, WithProviderClientFactory(factory))
models, err := fetcher.FetchModels(ctx, provider)

With options:

fetcher := NewProviderFetcher(providers,
    WithProviderClientFactory(factory),
    WithTimeout(30*time.Second),
)
models, err := fetcher.FetchModels(ctx, provider, WithAllowMissingAPIKey())

func (*ProviderFetcher) FetchRawResponse

func (pf *ProviderFetcher) FetchRawResponse(ctx context.Context, provider *catalogs.Provider, endpoint string, opts ...ProviderOption) ([]byte, *FetchStats, error)

FetchRawResponse fetches the raw API response from a provider's endpoint. This is useful for testing, debugging, or saving raw responses as testdata.

The endpoint parameter should be the full URL to the API endpoint. The response is returned as raw bytes (JSON) without any parsing, along with fetch statistics.

func (*ProviderFetcher) HasClient

func (pf *ProviderFetcher) HasClient(id catalogs.ProviderID) bool

HasClient checks if a provider ID has a client implementation.

func (*ProviderFetcher) List

func (pf *ProviderFetcher) List() []catalogs.ProviderID

List returns all provider IDs that have client implementations.

func (*ProviderFetcher) Providers

func (pf *ProviderFetcher) Providers() *catalogs.Providers

Providers returns the providers that can be used by the provider fetcher.

type ProviderOption

ProviderOption configures ProviderFetcher behavior.

type ProviderOption func(*providerOptions)

func WithAllowMissingAPIKey

func WithAllowMissingAPIKey() ProviderOption

WithAllowMissingAPIKey allows operations even when API key is not configured. Useful for checking provider support without credentials.

func WithProviderClientFactory

func WithProviderClientFactory(factory ProviderClientFactory) ProviderOption

WithProviderClientFactory configures the factory used to create provider API clients.

func WithProviderRawFetcher

func WithProviderRawFetcher(fetcher ProviderRawFetcher) ProviderOption

WithProviderRawFetcher configures the raw provider response fetcher.

func WithTimeout

func WithTimeout(d time.Duration) ProviderOption

WithTimeout sets a timeout for provider operations. The timeout applies to the context passed to FetchModels.

func WithoutCredentialLoading

func WithoutCredentialLoading() ProviderOption

WithoutCredentialLoading disables automatic credential loading from environment. Use this when credentials are already loaded or when testing.

type ProviderRawFetcher

ProviderRawFetcher fetches a raw provider API response.

type ProviderRawFetcher func(context.Context, *catalogs.Provider, string) (*RawFetchResult, error)

type RawFetchResult

RawFetchResult contains the result of a raw provider fetch operation.

type RawFetchResult struct {
    Data       []byte
    Response   *http.Response
    Latency    time.Duration
    RequestURL string
}

type ResourceType

ResourceType is a type alias for catalogmeta.ResourceType to maintain backward compatibility. This allows existing code to continue using sources.ResourceType while benefiting from the shared type definitions in pkg/catalogmeta.

type ResourceType = catalogmeta.ResourceType

type Revision

Revision identifies the exact upstream or normalized content revision.

type Revision = catalogmeta.ObservationRevision

type RevisionKind

RevisionKind identifies how an upstream observation revision was obtained.

type RevisionKind = catalogmeta.ObservationRevisionKind

type SchemaDriftDisposition

SchemaDriftDisposition defines how a mismatch or unknown member is handled.

type SchemaDriftDisposition string

const (
    // SchemaDriftRejectSource rejects a structurally unusable source observation.
    SchemaDriftRejectSource SchemaDriftDisposition = "reject_source"
    // SchemaDriftRejectRecord quarantines one malformed record and preserves valid siblings.
    SchemaDriftRejectRecord SchemaDriftDisposition = "reject_record"
    // SchemaDriftClassify preserves a fingerprint/evidence record for review without promotion.
    SchemaDriftClassify SchemaDriftDisposition = "classify"
    // SchemaDriftPreserve retains the exact value inside the source extension boundary.
    SchemaDriftPreserve SchemaDriftDisposition = "preserve"
    // SchemaDriftNotApplicable means the disposition does not apply at this path.
    SchemaDriftNotApplicable SchemaDriftDisposition = "n/a"
)

type SchemaDriftPolicy

SchemaDriftPolicy is the executable strict/tolerant contract for one path.

type SchemaDriftPolicy struct {
    Record       SchemaRecord
    Path         string
    Class        SchemaFieldClass
    Required     bool
    Mismatch     SchemaDriftDisposition
    UnknownField SchemaDriftDisposition
    Rationale    string
}

func SchemaDriftPolicies

func SchemaDriftPolicies(record SchemaRecord) []SchemaDriftPolicy

SchemaDriftPolicies returns caller-owned policies for a source record shape.

type SchemaFieldClass

SchemaFieldClass explains why a field boundary is strict or tolerant.

type SchemaFieldClass string

const (
    // SchemaFieldIdentity is required identity whose absence or type drift rejects the record.
    SchemaFieldIdentity SchemaFieldClass = "strict_identity"
    // SchemaFieldContainer is an object/array boundary whose type drift rejects its scope.
    SchemaFieldContainer SchemaFieldClass = "strict_container"
    // SchemaFieldValue is a known scalar value validated before canonical promotion.
    SchemaFieldValue SchemaFieldClass = "validated_value"
    // SchemaFieldExtension is an explicitly lossless source-extension boundary.
    SchemaFieldExtension SchemaFieldClass = "tolerant_extension"
)

type SchemaRecord

SchemaRecord identifies one independently validated source record shape.

type SchemaRecord string

const (
    // SchemaRecordObservation is the source observation envelope.
    SchemaRecordObservation SchemaRecord = "observation"
    // SchemaRecordCatalog is a complete decoded source catalog.
    SchemaRecordCatalog SchemaRecord = "catalog"
    // SchemaRecordProvider is one provider record inside a source catalog.
    SchemaRecordProvider SchemaRecord = "provider"
    // SchemaRecordModel is one source model record before canonical promotion.
    SchemaRecordModel SchemaRecord = "model"
    // SchemaRecordModelDefinition is one canonical provider-independent definition.
    SchemaRecordModelDefinition SchemaRecord = "model_definition"
    // SchemaRecordProviderOffering is one canonical provider-scoped offering.
    SchemaRecordProviderOffering SchemaRecord = "provider_offering"
)

type Source

Source observes catalog information from one configured upstream.

Implementations must be safe for repeated and concurrent Observe calls. Observe returns the complete result of that call directly and must not require a prior call or publish mutable result state through the Source.

type Source interface {
    // ID returns the stable identity of this source.
    ID() ID

    // Observe retrieves and returns one immutable source result directly. Calls
    // must not depend on prior Observe calls or publish result state on Source.
    Observe(ctx context.Context, opts ...Option) (Observation, error)

    // Cleanup releases resources after all Observe calls have completed.
    Cleanup() error

    // Dependencies returns the list of external dependencies this source requires
    Dependencies() []Dependency

    // IsOptional returns true if the sync can succeed without this source
    IsOptional() bool
}

Generated by gomarkdoc