sources

sources

Package sources provides abstractions for fetching AI model catalog data from various external sources including provider APIs and community repositories.

sources

import "github.com/agentstation/starmap/pkg/sources"

Package sources provides public APIs for working with AI model data sources.

Package sources defines interfaces and types for catalog data sources. Sources are responsible for fetching and synchronizing model data from various providers including local files, provider APIs, and external repositories.

The package provides a unified interface for different data sources while supporting merge strategies, authorities for data precedence, and flexible configuration options.

Example usage:

// Create a provider fetcher
fetcher := NewProviderFetcher(providers)

// Fetch models from a provider
models, err := fetcher.FetchModels(ctx, provider)
if err != nil {
    log.Fatal(err)
}

// Check if a provider is supported
if fetcher.HasClient(providerID) {
    // Provider has a client implementation
}

Index

• Constants
• type Dependency
• type DependencyStatus
• type FetchStats
  • func (s *FetchStats) HumanSize() string
• type ID
  • func IDs() []ID
• type Option
  • func WithCleanupRepo(cleanup bool) Option
  • func WithFresh(fresh bool) Option
  • func WithProviderFilter(providerID catalogs.ProviderID) Option
  • func WithReformat(reformat bool) Option
  • func WithSafeMode(safeMode bool) Option
• type Options
  • func Defaults() *Options
  • func (o *Options) Apply(opts ...Option) *Options
• type ProviderFetcher
  • func NewProviderFetcher(providers *catalogs.Providers, opts ...ProviderOption) *ProviderFetcher
  • func (pf *ProviderFetcher) FetchModels(ctx context.Context, provider *catalogs.Provider, opts ...ProviderOption) ([]catalogs.Model, error)
  • func (pf *ProviderFetcher) FetchRawResponse(ctx context.Context, provider *catalogs.Provider, endpoint string, opts ...ProviderOption) ([]byte, *FetchStats, error)
  • func (pf *ProviderFetcher) HasClient(id catalogs.ProviderID) bool
  • func (pf *ProviderFetcher) List() []catalogs.ProviderID
  • func (pf *ProviderFetcher) Providers() *catalogs.Providers
• type ProviderOption
  • func WithAllowMissingAPIKey() ProviderOption
  • func WithTimeout(d time.Duration) ProviderOption
  • func WithoutCredentialLoading() ProviderOption
• type ResourceType
• type Source
• type Sources
  • func NewSources() *Sources
  • func (s *Sources) Delete(id ID)
  • func (s *Sources) Get(id ID) (Source, bool)
  • func (s *Sources) IDs() []ID
  • func (s *Sources) Len() int
  • func (s *Sources) List() []Source
  • func (s *Sources) Set(id ID, src Source)

Constants

Common source identifiers - exported as package-level constants for convenience.

const (
    ProvidersID     = types.ProvidersID
    ModelsDevGitID  = types.ModelsDevGitID
    ModelsDevHTTPID = types.ModelsDevHTTPID
    LocalCatalogID  = types.LocalCatalogID
)

Common resource type identifiers - exported as package-level constants for convenience.

const (
    ResourceTypeModel    = types.ResourceTypeModel
    ResourceTypeProvider = types.ResourceTypeProvider
    ResourceTypeAuthor   = types.ResourceTypeAuthor
)

type Dependency

Dependency represents an external tool or runtime required by a source.

type Dependency struct {
    // Core identification
    Name        string // Machine name: "bun", "git", "docker"
    DisplayName string // Human-readable: "Bun JavaScript runtime"
    Required    bool   // false = source is optional or has fallback

    // Checking availability
    CheckCommands []string // Try in order: ["bun", "bunx"]
    MinVersion    string   // Optional: "1.0.0"

    // Installation
    InstallURL         string // https://bun.sh/docs/installation
    AutoInstallCommand string // Optional: "curl -fsSL https://bun.sh/install | bash"

    // User messaging
    Description       string // "Builds models.dev data locally (same as HTTP source)"
    WhyNeeded         string // "Required to build api.json from TypeScript source"
    AlternativeSource string // "models_dev_http provides same data without dependencies"
}

type DependencyStatus

DependencyStatus represents the availability status of a dependency.

type DependencyStatus struct {
    Available  bool   // Whether the dependency is available
    Version    string // Version string if available and detectable
    Path       string // Full path to executable if found
    CheckError error  // Error from check command if not available
}

type FetchStats

FetchStats contains metadata about a fetch operation. This provides transparency into API requests for debugging and monitoring.

type FetchStats struct {
    URL          string        // Endpoint that was called
    StatusCode   int           // HTTP response status code
    Latency      time.Duration // Request duration
    PayloadSize  int64         // Response body size in bytes
    ContentType  string        // Content-Type from response header
    AuthMethod   string        // How authentication was applied (Header, Query, None)
    AuthLocation string        // Where auth was placed (header name or query param name)
    AuthScheme   string        // Authentication scheme for header auth (Bearer, Basic, Direct)
}

func (*FetchStats) HumanSize

func (s *FetchStats) HumanSize() string

HumanSize returns the payload size in human-readable format.

type ID

ID represents the identifier of a data source. ID is a type alias for types.SourceID to maintain backward compatibility. This allows existing code to continue using sources.ID while benefiting from the shared type definitions in pkg/types.

type ID = types.SourceID

func IDs

func IDs() []ID

IDs returns all available source identifiers. Delegates to types.SourceIDs() to maintain consistency.

type Option

Option is a function that configures options.

type Option func(*Options)

func WithCleanupRepo

func WithCleanupRepo(cleanup bool) Option

WithCleanupRepo configures whether to clean up temporary repositories after fetch.

func WithFresh

func WithFresh(fresh bool) Option

WithFresh configures fresh sync mode for sources.

func WithProviderFilter

func WithProviderFilter(providerID catalogs.ProviderID) Option

WithProviderFilter configures filtering for a specific provider.

func WithReformat

func WithReformat(reformat bool) Option

WithReformat configures whether to reformat output files.

func WithSafeMode

func WithSafeMode(safeMode bool) Option

WithSafeMode configures safe mode for sources.

type Options

Options is the configuration for sources.

type Options struct {
    // Provider filtering (needed by provider source)
    ProviderID *catalogs.ProviderID

    // Behavior flags (needed by various sources)
    Fresh    bool // Fresh sync (delete existing before adding)
    SafeMode bool // Don't delete models, only add/update

    // Typed source-specific options
    CleanupRepo bool // For models.dev git source - remove repo after fetch
    Reformat    bool // For file-based sources - reformat output files
}

func Defaults

func Defaults() *Options

Defaults returns source options with default values.

func (*Options) Apply

func (o *Options) Apply(opts ...Option) *Options

Apply applies a set of options to create configured sourceOptions This is a helper for sources to use internally.

type ProviderFetcher

ProviderFetcher provides operations for fetching models from provider APIs. This is the public API for external packages to interact with provider data.

type ProviderFetcher struct {
    // contains filtered or unexported fields
}

func NewProviderFetcher

func NewProviderFetcher(providers *catalogs.Providers, opts ...ProviderOption) *ProviderFetcher

NewProviderFetcher creates a new provider fetcher for interacting with provider APIs. It provides a clean public interface for external packages. The providers parameter should contain the catalog providers to create clients for.

func (*ProviderFetcher) FetchModels

func (pf *ProviderFetcher) FetchModels(ctx context.Context, provider *catalogs.Provider, opts ...ProviderOption) ([]catalogs.Model, error)

FetchModels fetches available models from a single provider's API. It handles credential loading, client creation, and API communication.

Example:

fetcher := NewProviderFetcher()
models, err := fetcher.FetchModels(ctx, provider)

With options:

fetcher := NewProviderFetcher(WithTimeout(30 * time.Second))
models, err := fetcher.FetchModels(ctx, provider, WithAllowMissingAPIKey())

func (*ProviderFetcher) FetchRawResponse

func (pf *ProviderFetcher) FetchRawResponse(ctx context.Context, provider *catalogs.Provider, endpoint string, opts ...ProviderOption) ([]byte, *FetchStats, error)

FetchRawResponse fetches the raw API response from a provider's endpoint. This is useful for testing, debugging, or saving raw responses as testdata.

The endpoint parameter should be the full URL to the API endpoint. The response is returned as raw bytes (JSON) without any parsing, along with fetch statistics.

func (*ProviderFetcher) HasClient

func (pf *ProviderFetcher) HasClient(id catalogs.ProviderID) bool

HasClient checks if a provider ID has a client implementation.

func (*ProviderFetcher) List

func (pf *ProviderFetcher) List() []catalogs.ProviderID

List returns all provider IDs that have client implementations.

func (*ProviderFetcher) Providers

func (pf *ProviderFetcher) Providers() *catalogs.Providers

Providers returns the providers that can be used by the provider fetcher.

type ProviderOption

ProviderOption configures ProviderFetcher behavior.

type ProviderOption func(*providerOptions)

func WithAllowMissingAPIKey

func WithAllowMissingAPIKey() ProviderOption

WithAllowMissingAPIKey allows operations even when API key is not configured. Useful for checking provider support without credentials.

func WithTimeout

func WithTimeout(d time.Duration) ProviderOption

WithTimeout sets a timeout for provider operations. The timeout applies to the context passed to FetchModels.

func WithoutCredentialLoading

func WithoutCredentialLoading() ProviderOption

WithoutCredentialLoading disables automatic credential loading from environment. Use this when credentials are already loaded or when testing.

type ResourceType

ResourceType is a type alias for types.ResourceType to maintain backward compatibility. This allows existing code to continue using sources.ResourceType while benefiting from the shared type definitions in pkg/types.

type ResourceType = types.ResourceType

type Source

Source represents a data source for catalog information.

type Source interface {
    // Type returns the type of this source
    ID() ID

    // Name returns a human-friendly name for this source
    Name() string

    // Fetch retrieves data from this source
    // Sources handle their own concurrency internally
    Fetch(ctx context.Context, opts ...Option) error

    // Catalog returns the catalog of this source
    Catalog() catalogs.Catalog

    // Cleanup releases any resources (called after all Fetch operations)
    Cleanup() error

    // Dependencies returns the list of external dependencies this source requires
    Dependencies() []Dependency

    // IsOptional returns true if the sync can succeed without this source
    IsOptional() bool
}

type Sources

Sources is a thread-safe container for managing multiple data sources.

type Sources struct {
    // contains filtered or unexported fields
}

func NewSources

func NewSources() *Sources

NewSources creates a new Sources instance.

func (*Sources) Delete

func (s *Sources) Delete(id ID)

Delete deletes a source by ID.

func (*Sources) Get

func (s *Sources) Get(id ID) (Source, bool)

Get returns a source by ID.

func (*Sources) IDs

func (s *Sources) IDs() []ID

IDs returns a slice of all source IDs.

func (*Sources) Len

func (s *Sources) Len() int

Len returns the number of sources.

func (*Sources) List

func (s *Sources) List() []Source

List returns a slice of all sources.

func (*Sources) Set

func (s *Sources) Set(id ID, src Source)

Set sets a source by ID.

Generated by gomarkdoc