library

package
v0.63.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 9, 2026 License: AGPL-3.0 Imports: 39 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	RawPrefix          = "library/files"
	MaxRawListPageSize = 500
	// DefaultFSMinFreeBytes keeps a local deployment from consuming the final
	// disk reserve with immutable library snapshots.
	DefaultFSMinFreeBytes int64 = 5 << 30
)
View Source 
const (
	// TextParserProfile is persisted with every generation. Any setting that can
	// change chunk bytes or locators must produce a new profile value.
	TextParserProfile = "" /* 139-byte string literal not displayed */

	TextChunkRunes        = 1_000
	TextChunkOverlapRunes = 200

	MaxParsedChunks            = 32_768
	MaxParsedChunkContentBytes = 40 << 20
	MaxStagedChunksPerTx       = 500
	MaxStagedContentBytesPerTx = 2 << 20
)
View Source 
const (
	MaxFileBytes = 25 << 20

	SystemMaxFiles      int64 = 4_000
	SystemMaxBytes      int64 = 20 << 30
	SystemAgentMaxFiles int64 = 1_000
	SystemAgentMaxBytes int64 = 5 << 30
	PersonalMaxFiles    int64 = 2_000
	PersonalMaxBytes    int64 = 10 << 30
)
View Source 
const (
	MediaTypeMarkdown = "text/markdown"
	MediaTypeText     = "text/plain"
)
View Source 
const LibraryQueue = "stella_library"

Variables

View Source 
var (
	ErrRawAlreadyExists    = errors.New("library raw object already exists")
	ErrRawStorageDegraded  = errors.New("library raw storage is degraded")
	ErrInvalidRawStorePage = errors.New("invalid library raw store page")
)
View Source 
var (
	ErrNoExtractedText   = errors.New("document contains no extractable text")
	ErrInvalidParserData = errors.New("parser returned invalid chunk data")
	ErrParseResultLimit  = errors.New("parser result exceeds the configured limit")
)
View Source 
var (
	ErrInvalidOwner        = errors.New("invalid library owner")
	ErrFileTooLarge        = errors.New("library file is too large")
	ErrUnsupportedFileType = errors.New("unsupported library file type")
	ErrInvalidFile         = errors.New("invalid library file")
	ErrQuotaExceeded       = errors.New("library quota exceeded")
	ErrNotFound            = errors.New("library file not found")
	ErrForbidden           = errors.New("library access forbidden")
	ErrServiceUnavailable  = errors.New("library service is unavailable")
	ErrSpoolCapacity       = errors.New("library upload spool is at capacity")
	ErrGenerationConflict  = errors.New("library chunk generation identity conflicts with durable state")
	ErrGenerationChanged   = errors.New("library chunk generation state changed")
	ErrRawIntegrity        = errors.New("library raw snapshot failed integrity validation")
)

Functions

func FileIDFromRawKey 

func FileIDFromRawKey(key string) (string, error)

FileIDFromRawKey validates a canonical key before reconciliation uses it as a database identity. Malformed objects are retained, never guessed at.

func RawKey 

func RawKey(fileID string) (string, error)

RawKey derives the only canonical object key for a LibraryFile.

Types

type ChunkLocator 

type ChunkLocator struct {
	FirstPage   *uint32  `json:"first_page,omitempty"`
	LastPage    *uint32  `json:"last_page,omitempty"`
	HeadingPath []string `json:"heading_path,omitempty"`
	ByteStart   int      `json:"byte_start"`
	ByteEnd     int      `json:"byte_end"`
}

ChunkLocator records stable source positioning. Byte offsets are retained for internal diagnostics but are removed before a chunk is returned to a model.

type ChunkSetStatus 

type ChunkSetStatus string

ChunkSetStatus is the publication state of one deterministic generation. Only a ready set referenced by LibraryFile.ActiveChunkSetID may be retrieved. 

const (
	ChunkSetStatusBuilding ChunkSetStatus = "building"
	ChunkSetStatusReady    ChunkSetStatus = "ready"
	ChunkSetStatusFailed   ChunkSetStatus = "failed"
)

type FSRawStore 

type FSRawStore struct {
	// contains filtered or unexported fields
}

func NewFSRawStore 

func NewFSRawStore(root string, minFreeBytes int64) (*FSRawStore, error)

func (*FSRawStore) Create 

func (s *FSRawStore) Create(ctx context.Context, key string, reader io.Reader) error

func (*FSRawStore) Delete 

func (s *FSRawStore) Delete(ctx context.Context, key string) error

func (*FSRawStore) ListPage 

func (s *FSRawStore) ListPage(
	ctx context.Context,
	prefix, cursor string,
	limit int,
) (RawPage, error)

func (*FSRawStore) Open 

func (s *FSRawStore) Open(ctx context.Context, key string) (io.ReadCloser, error)

func (*FSRawStore) SupportsOrphanCollection 

func (*FSRawStore) SupportsOrphanCollection() bool

SupportsOrphanCollection reports that the local Library root belongs to this Stella deployment rather than a shared external namespace.

type FileStatus 

type FileStatus string

FileStatus is the durable processing state of one immutable snapshot. 

const (
	FileStatusProcessing FileStatus = "processing"
	FileStatusReady      FileStatus = "ready"
	FileStatusFailed     FileStatus = "failed"
)

type LibraryFile 

type LibraryFile struct {
	ID               string
	Owner            Owner
	FileName         string
	MediaType        string
	SizeBytes        int64
	RawSHA256        []byte
	Status           FileStatus
	ErrorMessage     string
	ActiveChunkSetID string
	DeletedAt        *time.Time
	CreatedAt        time.Time
	UpdatedAt        time.Time
}

LibraryFile is internal metadata for one canonical immutable source snapshot. Raw bytes remain in RawStore and are never included in this value.

type Owner 

type Owner struct {
	Scope   Scope
	UserID  string
	AgentID string
}

Owner is the normalized four-part scope tuple. Empty IDs are database NULLs.

func (Owner) Validate 

func (o Owner) Validate() error

Validate enforces the same four legal combinations as the database CHECK.

type ParsedChunk 

type ParsedChunk struct {
	Content string
	Locator ChunkLocator
}

ParsedChunk is the normalized parser result persisted as a Library chunk.

type Parser 

type Parser interface {
	Parse(ctx context.Context, path, mediaType string) ([]ParsedChunk, error)
}

Parser is the bounded document parser used by the asynchronous chunk worker.

type Quota 

type Quota struct {
	UsedFiles int64
	MaxFiles  int64
	UsedBytes int64
	MaxBytes  int64
}

Quota describes current usage and the fixed limit of one logical quota pool.

func (Quota) CanAdd 

func (q Quota) CanAdd(sizeBytes int64) bool

CanAdd reports whether adding one immutable file stays inside both limits.

type QuotaExceededError 

type QuotaExceededError struct {
	Quota Quota
}

QuotaExceededError carries the authoritative pool state at rejection time.

func (*QuotaExceededError) Error 

func (e *QuotaExceededError) Error() string

func (*QuotaExceededError) Unwrap 

func (e *QuotaExceededError) Unwrap() error

type RawObject 

type RawObject struct {
	Key          string
	Size         int64
	LastModified time.Time
}

RawObject is the minimum storage metadata required by bounded orphan GC.

type RawPage 

type RawPage struct {
	Objects    []RawObject
	NextCursor string
}

RawPage is one bounded canonical-key-ordered enumeration page. NextCursor is the last returned key and is an exclusive lower bound for the following page.

type RawStore 

type RawStore interface {
	Create(ctx context.Context, key string, reader io.Reader) error
	Open(ctx context.Context, key string) (io.ReadCloser, error)
	Delete(ctx context.Context, key string) error
	ListPage(ctx context.Context, prefix, cursor string, limit int) (RawPage, error)
	// SupportsOrphanCollection is true only when RawPrefix is owned exclusively
	// by this Stella deployment, so unknown objects can be deleted safely.
	SupportsOrphanCollection() bool
}

RawStore owns canonical immutable raw snapshots. Create must never replace an existing key. ListPage must order by canonical key and bound both returned objects and adapter memory; it does not provide snapshot isolation across concurrent pages.

func NewRawStoreFromConfig 

func NewRawStoreFromConfig(
	root string,
	deploymentS3 config.BlobS3Config,
	options RawStoreOptions,
) (RawStore, error)

NewRawStoreFromConfig selects local FS or deployment S3 using the existing STELLA_BLOB_S3_* configuration group.

type RawStoreOptions 

type RawStoreOptions struct {
	TempDir        string
	FSMinFreeBytes int64
	S3Admission    func(context.Context) error
}

RawStoreOptions holds backend admission knobs without exposing a second Library-specific storage configuration group.

type S3RawStore 

type S3RawStore struct {
	// contains filtered or unexported fields
}

func NewS3RawStore 

func NewS3RawStore(
	config blob.S3Config,
	tempDir string,
	admission func(context.Context) error,
) (*S3RawStore, error)

func (*S3RawStore) Create 

func (s *S3RawStore) Create(ctx context.Context, key string, reader io.Reader) error

func (*S3RawStore) Delete 

func (s *S3RawStore) Delete(ctx context.Context, key string) error

func (*S3RawStore) ListPage 

func (s *S3RawStore) ListPage(
	ctx context.Context,
	prefix, cursor string,
	limit int,
) (RawPage, error)

func (*S3RawStore) Open 

func (s *S3RawStore) Open(ctx context.Context, key string) (io.ReadCloser, error)

func (*S3RawStore) SupportsOrphanCollection 

func (*S3RawStore) SupportsOrphanCollection() bool

SupportsOrphanCollection remains false until S3 keys carry a deployment namespace or another ownership marker. Exact-key tombstone cleanup is still safe and continues to use Delete directly.

type Scope 

type Scope string

Scope is the owner scope shared by Library management and retrieval. 

const (
	ScopeSystem      Scope = "system"
	ScopeSystemAgent Scope = "system_agent"
	ScopeUser        Scope = "user"
	ScopeUserAgent   Scope = "user_agent"
)

type Service 

type Service struct {
	// contains filtered or unexported fields
}

Service owns authorization, bounded acquisition, immutable raw publication, and the short metadata-plus-job transaction.

func NewService 

func NewService(config ServiceConfig) (*Service, error)

func (*Service) BindRiverClient 

func (s *Service) BindRiverClient(client *river.Client[pgx.Tx]) error

BindRiverClient injects the single shared working River client. Tests may still supply an insert-only client in ServiceConfig; production binds once before the client starts.

func (*Service) CreateManagedUpload 

func (s *Service) CreateManagedUpload(
	ctx context.Context,
	authority authz.Authority,
	scope Scope,
	agentID string,
	fileName string,
	source io.Reader,
) (LibraryFile, error)

CreateManagedUpload is the upload acquisition boundary. Scope and Agent authorization finish before prepareUpload consumes a single source byte.

func (*Service) DeleteManaged 

func (s *Service) DeleteManaged(
	ctx context.Context,
	authority authz.Authority,
	id string,
) error

DeleteManaged commits the tombstone before attempting cancellation or raw cleanup. The durable tombstone is the visibility guarantee; cancellation is only a best-effort resource optimization.

func (*Service) Get 

func (s *Service) Get(ctx context.Context, id string) (LibraryFile, error)

Get returns live internal metadata only; raw bytes are never exposed.

func (*Service) GetManaged 

func (s *Service) GetManaged(
	ctx context.Context,
	authority authz.Authority,
	id string,
) (LibraryFile, error)

GetManaged loads a file and authorizes the management view against its durable owner tuple. Every denial is opaque so knowing a foreign UUID cannot reveal whether that file exists.

func (*Service) QueueConfig 

func (s *Service) QueueConfig() (string, river.QueueConfig)

QueueConfig returns the dedicated per-node parser/maintenance concurrency.

func (*Service) RegisterRiverWorkers 

func (s *Service) RegisterRiverWorkers(workers *river.Workers)

RegisterRiverWorkers contributes all internal Library workers to the one process-wide River client. No management or Agent surface is registered here.

func (*Service) ResolveManageOwner 

func (s *Service) ResolveManageOwner(
	ctx context.Context,
	authority authz.Authority,
	scope Scope,
	agentID string,
) (Owner, error)

ResolveManageOwner authorizes a scope-keyed management operation and returns the exact owner tuple that may be passed to List or Create. HTTP parameters select a scope; they never supply the owning user, which always comes from the trusted Authority.

func (*Service) StartReconciliation 

func (s *Service) StartReconciliation() (rivertype.PeriodicJobHandle, error)

StartReconciliation registers one leader-elected bounded repair chain.

func (*Service) StopReconciliation 

func (s *Service) StopReconciliation(handle rivertype.PeriodicJobHandle)

type ServiceConfig 

type ServiceConfig struct {
	DB       *pgxpool.Pool
	RawStore RawStore
	Parser   Parser
	// ParserProfile is an immutable identity for every setting that can affect
	// generated chunks. Callers must change it whenever parser behavior changes.
	ParserProfile            string
	River                    *river.Client[pgx.Tx]
	Logger                   *slog.Logger
	TempDir                  string
	MaxConcurrentUploads     int
	MaxSpoolBytes            int64
	AgentAccess              *agentaccess.Service
	SnapshotCommitTimeout    time.Duration
	DatabaseStatementTimeout time.Duration
	DatabaseLockTimeout      time.Duration
	ReconciliationInterval   time.Duration
	StaleDerivationAfter     time.Duration
	OrphanMinAge             time.Duration
	MaxClockSkew             time.Duration
	OrphanSafetyMargin       time.Duration
	MaxWorkers               int
}

ServiceConfig contains the internal Library ingestion and lifecycle dependencies. Public management and retrieval surfaces are composed later.

type TextParser 

type TextParser struct{}

TextParser deterministically chunks UTF-8 text and Markdown without an external runtime. Markdown is intentionally treated as plain source text in V1; richer format-specific parsing can introduce a separate profile later.

func NewTextParser 

func NewTextParser() *TextParser

func (*TextParser) Parse 

func (*TextParser) Parse(ctx context.Context, filePath, mediaType string) ([]ParsedChunk, error)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.