Documentation
¶
Overview ¶
Package dynamo implements an EventStore backed by DynamoDB and related utilities.
Index ¶
- Constants
- Variables
- func ClassifyError(err error) policy.Class
- func FromAttributeValue(from lambdaevents.DynamoDBAttributeValue) (types.AttributeValue, error)
- func FromAttributeValueList(from []lambdaevents.DynamoDBAttributeValue) (to []types.AttributeValue, err error)
- func FromAttributeValueMap(from map[string]lambdaevents.DynamoDBAttributeValue) (to map[string]types.AttributeValue, err error)
- func HasConditionalCheckFailure(err error) (bool, int)
- func IsInvalidEventError(err error) bool
- func MarshalJSON(item map[string]lambdaevents.DynamoDBAttributeValue) (evt.SerializedEvent, []byte, error)
- func MarshalViewToItem(encoder *attributevalue.Encoder, view *evt.SerializedView) (map[string]types.AttributeValue, error)
- func NewViewProjector(tableName string) evt.EventProjector
- func NewViewRepository(client Client, viewsTable string) evt.ViewRepository
- func UnmarshalAttributeMap[T any](value map[string]lambdaevents.DynamoDBAttributeValue, out T) error
- type Client
- type Event
- type ExpressionStreamer
- type HeadStore
- func (h *HeadStore) Backfill(ctx context.Context, source evt.EntityHeadStreamer, entityType evt.EntityType) (int, error)
- func (h *HeadStore) Name() string
- func (h *HeadStore) Process(ctx context.Context, records []projectors.StreamRecord) ([]projectors.BatchItemFailure, error)
- func (h *HeadStore) StreamEntityHeads(ctx context.Context, entityType evt.EntityType) (map[evt.EntityID]evt.EventSequence, error)
- func (h *HeadStore) StreamEntityHeadsFunc(ctx context.Context, entityType evt.EntityType, ...) error
- func (h *HeadStore) WithConsistentRead(consistent bool) *HeadStore
- type InvalidEventError
- type Repository
- func (repo *Repository) Commit(ctx context.Context, result evt.SerializedResult) error
- func (repo *Repository) CommitStream(ctx context.Context, channel <-chan result.Result[evt.SerializedResult]) []error
- func (repo *Repository) CommitWithSnapshot(ctx context.Context, result evt.SerializedResult, entityType evt.EntityType, ...) error
- func (repo *Repository) CompactBelow(ctx context.Context, entityID evt.EntityID, throughSequence evt.EventSequence) (int, error)
- func (repo *Repository) Delete(ctx context.Context, serializedEvents []evt.SerializedEvent) error
- func (repo *Repository) GetEvents(ctx context.Context, entityID evt.EntityID) ([]evt.SerializedEvent, error)
- func (repo *Repository) GetLatestEvents(ctx context.Context, entityID evt.EntityID, lastSequence evt.EventSequence) ([]evt.SerializedEvent, error)
- func (repo *Repository) GetMaxSequence(ctx context.Context, entityID evt.EntityID) (evt.EventSequence, error)
- func (repo *Repository) GetSnapshot(ctx context.Context, entityID evt.EntityID) (*evt.SerializedSnapshot, error)
- func (repo *Repository) PutSnapshot(ctx context.Context, entityType evt.EntityType, entityID evt.EntityID, ...) error
- func (repo *Repository) RebuildProjectionsByQuery(ctx context.Context, opts StreamByQueryOptions, ...) (*evt.RebuildResult, error)
- func (repo *Repository) StreamAllEvents(ctx context.Context, filter evt.StreamFilter) <-chan result.Result[[]evt.SerializedEvent]
- func (repo *Repository) StreamAllEventsByExpression(ctx context.Context, expr *expression.Expression) <-chan result.Result[[]evt.SerializedEvent]
- func (repo *Repository) StreamEntities(ctx context.Context, filter evt.StreamFilter, ...) <-chan result.Result[evt.Entity]
- func (repo *Repository) StreamEntitiesByExpression(ctx context.Context, expr *expression.Expression, ...) <-chan result.Result[evt.Entity]
- func (repo *Repository) StreamEntitiesByQuery(ctx context.Context, opts StreamByQueryOptions, ...) <-chan result.Result[evt.Entity]
- func (repo *Repository) StreamEntitiesFromSnapshots(ctx context.Context, entityType evt.EntityType, seedEntity evt.SnapshotSeeder, ...) <-chan result.Result[evt.Entity]
- func (repo *Repository) StreamEntitiesFromSnapshotsWithOptions(ctx context.Context, opts StreamFromSnapshotsOptions, ...) <-chan result.Result[evt.Entity]
- func (repo *Repository) StreamEntityHeads(ctx context.Context, entityType evt.EntityType) (map[evt.EntityID]evt.EventSequence, error)
- func (repo *Repository) WithClock(now func() time.Time) *Repository
- func (repo *Repository) WithConsistentRead(consistent bool) *Repository
- func (repo *Repository) WithLogger(logger *slog.Logger) *Repository
- func (repo *Repository) WithRetention(retention Retention) *Repository
- func (repo *Repository) WithScanSegments(n int) *Repository
- type Retention
- type Snapshot
- type StreamByQueryOptions
- type StreamFromSnapshotsOptions
- type Transaction
- type TransactionGroup
- type View
- type ViewProjector
- type ViewPutGroup
- func (g *ViewPutGroup) HandleError(err error, _ int) error
- func (g *ViewPutGroup) Len() int
- func (g *ViewPutGroup) Merge(with evt.TransactionGroup) (evt.TransactionGroup, error)
- func (g *ViewPutGroup) MergeDynamo(with TransactionGroup) (TransactionGroup, error)
- func (g *ViewPutGroup) StorageType() evt.StorageType
- func (g *ViewPutGroup) ToWriteItems() []types.TransactWriteItem
- func (g *ViewPutGroup) TransactionType() evt.TransactionType
- type ViewRepository
- func (repo *ViewRepository) BatchGetViews(ctx context.Context, pks []string) ([]*evt.SerializedView, error)
- func (repo *ViewRepository) BatchGetViewsAtSK(ctx context.Context, pks []string, sk string) ([]*evt.SerializedView, error)
- func (repo *ViewRepository) DeleteView(ctx context.Context, pk string, sk string) error
- func (repo *ViewRepository) GetView(ctx context.Context, pk string) (*evt.SerializedView, error)
- func (repo *ViewRepository) ListViewsByEntityType(ctx context.Context, entityType evt.EntityType) ([]*evt.SerializedView, error)
- func (repo *ViewRepository) ListViewsByEntityTypeEach(ctx context.Context, entityType evt.EntityType, ...) error
- func (repo *ViewRepository) ListViewsByEntityTypePaged(ctx context.Context, entityType evt.EntityType, limit int, cursor string) (*evt.PagedResult, error)
- func (repo *ViewRepository) ListViewsByPK(ctx context.Context, pk string) ([]*evt.SerializedView, error)
- func (repo *ViewRepository) ListViewsByPKEach(ctx context.Context, pk string, fn func(*evt.SerializedView) error) error
- func (repo *ViewRepository) ListViewsByPKPaged(ctx context.Context, pk string, limit int, cursor string) (*evt.PagedResult, error)
- func (repo *ViewRepository) PutView(ctx context.Context, view *evt.SerializedView) error
- func (repo *ViewRepository) PutViews(ctx context.Context, views []*evt.SerializedView) error
Constants ¶
const DefaultHeadsProjectorName = "entity-heads"
DefaultHeadsProjectorName is the stable projector name used for idempotency keys and telemetry.
const FirstSnapshotCondition = "attribute_not_exists(seq)"
FirstSnapshotCondition guards the first snapshot put in a commit (snapshot sequence 1): no snapshot exists yet.
const FloorCondition = "attribute_not_exists(eventSeq) OR eventSeq < :first"
FloorCondition rejects a commit whose first event is at or below the stored snapshot's eventSeq. Compaction deletes the events a snapshot covers, so attribute_not_exists(sk) alone would let a writer that loaded before the snapshot recreate a compacted sequence.
const NextSnapshotCondition = "seq = :seq AND eventSeq < :first"
NextSnapshotCondition guards later snapshot puts in a commit: the stored snapshot is still the one the writer loaded (:seq), and the commit starts above its floor (:first).
const StorageType evt.StorageType = "DynamoDB"
StorageType indicates the type of storage used to execute Dynamo transactions
Variables ¶
var ErrSnapshotRaceCondition = errors.New("race condition - sequence number updated since it was read, try again")
ErrSnapshotRaceCondition is returned when a snapshot's sequence number was updated by another transaction between read and write. Callers can retry with a fresh entity load.
Functions ¶
func ClassifyError ¶
ClassifyError maps known DynamoDB backend errors into transient/permanent classes.
func FromAttributeValue ¶
func FromAttributeValue(from lambdaevents.DynamoDBAttributeValue) (types.AttributeValue, error)
FromAttributeValue converts a Lambda Event DynamoDB AttributeValue, including all nested members, to a dynamodbstreams AttributeValue.
func FromAttributeValueList ¶
func FromAttributeValueList(from []lambdaevents.DynamoDBAttributeValue) (to []types.AttributeValue, err error)
FromAttributeValueList converts a slice of Lambda Event DynamoDB AttributeValues, including all nested members, to a slice ofdynamodbstreams AttributeValue.
func FromAttributeValueMap ¶
func FromAttributeValueMap(from map[string]lambdaevents.DynamoDBAttributeValue) (to map[string]types.AttributeValue, err error)
FromAttributeValueMap converts a map of Lambda Event DynamoDB AttributeValues, including all nested members, to a dynamodbstreams map of AttributeValue.
func HasConditionalCheckFailure ¶
HasConditionalCheckFailure checks if the error returned by DynamoDB includes a conditional check failure.
func IsInvalidEventError ¶
IsInvalidEventError returns true when err wraps an InvalidEventError.
func MarshalJSON ¶
func MarshalJSON(item map[string]lambdaevents.DynamoDBAttributeValue) (evt.SerializedEvent, []byte, error)
MarshalJSON converts a DynamoDB attribute map to a serialized event and its JSON encoding.
func MarshalViewToItem ¶
func MarshalViewToItem(encoder *attributevalue.Encoder, view *evt.SerializedView) (map[string]types.AttributeValue, error)
MarshalViewToItem converts a serialized view to a DynamoDB attribute map. If the SK field is empty, it defaults to DefaultViewSK ("VIEW") to prevent DynamoDB validation errors for empty key attributes.
func NewViewProjector ¶
func NewViewProjector(tableName string) evt.EventProjector
NewViewProjector creates a new view projector for the given table name.
func NewViewRepository ¶
func NewViewRepository(client Client, viewsTable string) evt.ViewRepository
NewViewRepository constructs a view repository for the given table.
func UnmarshalAttributeMap ¶
func UnmarshalAttributeMap[T any](value map[string]lambdaevents.DynamoDBAttributeValue, out T) error
UnmarshalAttributeMap unmarshals a map of `DynamoDBAttributeValues“ (from stream events) into a Go value type
Types ¶
type Client ¶
type Client interface {
TransactWriteItems(
ctx context.Context,
params *dynamodb.TransactWriteItemsInput,
options ...func(*dynamodb.Options),
) (*dynamodb.TransactWriteItemsOutput, error)
BatchWriteItem(
ctx context.Context,
params *dynamodb.BatchWriteItemInput,
options ...func(*dynamodb.Options),
) (*dynamodb.BatchWriteItemOutput, error)
Query(
ctx context.Context,
params *dynamodb.QueryInput,
options ...func(*dynamodb.Options),
) (*dynamodb.QueryOutput, error)
Scan(
ctx context.Context,
params *dynamodb.ScanInput,
options ...func(*dynamodb.Options),
) (*dynamodb.ScanOutput, error)
GetItem(
ctx context.Context,
params *dynamodb.GetItemInput,
options ...func(*dynamodb.Options),
) (*dynamodb.GetItemOutput, error)
BatchGetItem(
ctx context.Context,
params *dynamodb.BatchGetItemInput,
options ...func(*dynamodb.Options),
) (*dynamodb.BatchGetItemOutput, error)
PutItem(
ctx context.Context,
params *dynamodb.PutItemInput,
options ...func(*dynamodb.Options),
) (*dynamodb.PutItemOutput, error)
DeleteItem(
ctx context.Context,
params *dynamodb.DeleteItemInput,
options ...func(*dynamodb.Options),
) (*dynamodb.DeleteItemOutput, error)
UpdateItem(
ctx context.Context,
params *dynamodb.UpdateItemInput,
options ...func(*dynamodb.Options),
) (*dynamodb.UpdateItemOutput, error)
ExportTableToPointInTime(
ctx context.Context,
params *dynamodb.ExportTableToPointInTimeInput,
options ...func(*dynamodb.Options),
) (*dynamodb.ExportTableToPointInTimeOutput, error)
}
Client is the subset of DynamoDB operations used by evt's DynamoDB backend.
type Event ¶
type Event struct {
PK evt.EntityID `json:"pk"` // the partition key is the entity ID
SK evt.EventSequence `json:"sk"` // the sort key is based on the event sequence
Type evt.EventType `json:"type"`
Version evt.EventVersion `json:"version"`
EntityType evt.EntityType `json:"entityType"`
Payload string `json:"payload"`
Metadata string `json:"metadata"`
// TTL is the optional DynamoDB time-to-live expiry (Unix epoch seconds). It is written only for
// entity types covered by a Repository retention policy; the omitempty tag drops it otherwise, so
// un-policed events carry no ttl attribute and are never auto-expired. See Repository.WithRetention.
TTL int64 `json:"ttl,omitempty"`
}
Event is a struct to serialize an Event for DynamoDB
type ExpressionStreamer ¶ added in v0.6.0
type ExpressionStreamer interface {
// StreamAllEventsByExpression scans all Events in the table, optionally filtered by a raw
// DynamoDB expression. A nil expression streams every event.
StreamAllEventsByExpression(
ctx context.Context,
expr *expression.Expression,
) <-chan result.Result[[]evt.SerializedEvent]
// StreamEntitiesByExpression streams reconstituted Entities, optionally filtered by a raw
// DynamoDB expression. A nil expression streams every entity.
StreamEntitiesByExpression(
ctx context.Context,
expr *expression.Expression,
applyEvent func(context.Context, evt.SerializedEvent, evt.Entity) (evt.Entity, error),
) <-chan result.Result[evt.Entity]
}
ExpressionStreamer is the DynamoDB-specific streaming capability for callers that need richer server-side filtering than the backend-neutral evt.StreamFilter exposes. The core evt.Repository contract intentionally carries no DynamoDB types, so callers that want to push a raw DynamoDB filter expression into the table scan type-assert a repository to ExpressionStreamer.
Like evt's other optional capabilities (Compactor, SnapshotStreamer), it is detected via a type assertion (repo.(dynamo.ExpressionStreamer)) and is not part of the backend-neutral contract a future SQL backend must satisfy.
type HeadStore ¶ added in v0.4.0
type HeadStore struct {
// contains filtered or unexported fields
}
HeadStore maintains one monotonic event head per entity for incremental rebuilds. A stream projector updates it asynchronously, so it adds no commit-path work and tolerates duplicate or out-of-order delivery. As an evt.EntityHeadStreamer, it scans the heads table instead of the log.
func NewHeadStore ¶ added in v0.4.0
NewHeadStore builds a HeadStore over the given heads table. Reads default to eventually consistent (half the RCU cost): a stale read only defers an entity's reprojection to the next rebuild, never skips it. Derive a strongly consistent variant with WithConsistentRead(true) when a read must reflect the latest projector write.
func (*HeadStore) Backfill ¶ added in v0.4.0
func (h *HeadStore) Backfill( ctx context.Context, source evt.EntityHeadStreamer, entityType evt.EntityType, ) (int, error)
Backfill seeds the heads table from a source of entity heads — typically a Repository whose StreamEntityHeads scans the event log — for entities that predate the heads projector. It upserts each head with the same monotonic condition as live projection, so it is safe to run concurrently with the projector and safe to re-run. Pass a non-empty entityType to scope the source read and to record that type on the written rows; call once per type to fully populate the type attribute. It returns the number of heads written.
func (*HeadStore) Process ¶ added in v0.4.0
func (h *HeadStore) Process( ctx context.Context, records []projectors.StreamRecord, ) ([]projectors.BatchItemFailure, error)
Process implements projectors.Projector, upserting each record's entity head. A failed upsert is reported as a per-record partial-batch failure so Lambda retries just that record; because the upsert is monotonic and idempotent, a retry (or a re-delivery) is always safe.
func (*HeadStore) StreamEntityHeads ¶ added in v0.4.0
func (h *HeadStore) StreamEntityHeads( ctx context.Context, entityType evt.EntityType, ) (map[evt.EntityID]evt.EventSequence, error)
StreamEntityHeads implements evt.EntityHeadStreamer by scanning the heads table. The heads table holds one small row per entity, so this reads only a few MB regardless of event-log size — the cheap change-detection path. The scan is eventually consistent by default (see NewHeadStore); use WithConsistentRead(true) for a strongly consistent read. entityType, when non-empty, restricts the result to that type.
func (*HeadStore) StreamEntityHeadsFunc ¶ added in v0.5.0
func (h *HeadStore) StreamEntityHeadsFunc( ctx context.Context, entityType evt.EntityType, visit func(evt.EntityID, evt.EventSequence) error, ) error
StreamEntityHeadsFunc implements evt.EntityHeadVisitor: it pages the heads table and invokes visit once per row while holding at most one page in memory. A rebuild using this method has a memory ceiling independent of entity count. The map returned by StreamEntityHeads is O(entities).
The heads table has one row per entity, so enumeration requires no deduplication set and retains only one page at a time. The scan is eventually consistent by default (see NewHeadStore); use WithConsistentRead(true) for a strongly consistent read. entityType, when non-empty, restricts enumeration to that type.
func (*HeadStore) WithConsistentRead ¶ added in v0.4.0
WithConsistentRead returns a shallow copy of the HeadStore with the consistent-read setting updated. When true, StreamEntityHeads uses strongly consistent reads; when false (the default), it uses eventually consistent reads at half the RCU cost. The original HeadStore is not modified, so a strong- and an eventual-consistency variant can be derived from the same base.
type InvalidEventError ¶
InvalidEventError indicates an irrecoverable validation/parsing error in a stream event. Callers can treat this as a non-retryable drop path.
func (*InvalidEventError) Error ¶
func (e *InvalidEventError) Error() string
func (*InvalidEventError) Unwrap ¶
func (e *InvalidEventError) Unwrap() error
type Repository ¶
type Repository struct {
EventsTable string
// contains filtered or unexported fields
}
Repository wires DynamoDB access for the event store.
func NewRepository ¶
func NewRepository(client Client, eventsTable string) *Repository
NewRepository constructs a Repository with configured encoders/decoders.
func (*Repository) Commit ¶
func (repo *Repository) Commit( ctx context.Context, result evt.SerializedResult, ) error
Commit writes serialized events to the events table.
func (*Repository) CommitStream ¶
func (repo *Repository) CommitStream( ctx context.Context, channel <-chan result.Result[evt.SerializedResult], ) []error
CommitStream commits a stream of events using batches under the 100-item limit. Each batch carries one floor check per entity (see FloorCondition), built when the batch is flushed: a transaction cannot touch an entity's snapshot item twice.
func (*Repository) CommitWithSnapshot ¶
func (repo *Repository) CommitWithSnapshot( ctx context.Context, result evt.SerializedResult, entityType evt.EntityType, entityID evt.EntityID, payload []byte, currentSnapshot evt.EventSequence, ) error
CommitWithSnapshot commits events and a snapshot in a single transaction.
func (*Repository) CompactBelow ¶ added in v0.2.0
func (repo *Repository) CompactBelow( ctx context.Context, entityID evt.EntityID, throughSequence evt.EventSequence, ) (int, error)
CompactBelow truncates an entity's event log by deleting events whose sequence is in the range [1, throughSequence], but only after verifying that a durable snapshot exists whose recorded EventSequence covers (>=) throughSequence. This guarantees every deleted event is already captured by the snapshot, so load and snapshot-aware rebuild remain correct.
The sk=0 snapshot row is never deleted. CompactBelow returns the number of events deleted. It returns evt.ErrCompactionUncovered (wrapped) when no snapshot exists or the snapshot does not cover throughSequence. A throughSequence < 1 is a no-op.
Concurrency: compaction only removes low, immutable, already-snapshotted events. Concurrent command handlers only append higher sequences and advance the sk=0 snapshot forward, so they never touch the deleted range; the coverage check is therefore safe without a transaction.
func (*Repository) Delete ¶
func (repo *Repository) Delete( ctx context.Context, serializedEvents []evt.SerializedEvent, ) error
Delete removes serialized events from the events table by point-deleting each (pk, sk). It performs NO snapshot-safety checks and can leave a stream unreplayable, so it is intended only for local and staging fixtures (e.g. resetting test data).
This method is compiled out of production builds: it is guarded by `//go:build !prod`, so a binary built with `-tags prod` does not contain it and cannot call it. For principled, snapshot-verified log truncation in any environment, use CompactBelow (evt.Compactor) instead.
WARNING: Use only in local and staging environments.
func (*Repository) GetEvents ¶
func (repo *Repository) GetEvents( ctx context.Context, entityID evt.EntityID, ) ([]evt.SerializedEvent, error)
GetEvents returns all events for an entity instance.
func (*Repository) GetLatestEvents ¶
func (repo *Repository) GetLatestEvents( ctx context.Context, entityID evt.EntityID, lastSequence evt.EventSequence, ) ([]evt.SerializedEvent, error)
GetLatestEvents returns events after the given sequence for an entity instance.
func (*Repository) GetMaxSequence ¶
func (repo *Repository) GetMaxSequence( ctx context.Context, entityID evt.EntityID, ) (evt.EventSequence, error)
GetMaxSequence returns the highest event sequence for an entity without fetching all events. Uses a reverse-order query with limit 1 for efficiency. Returns 0 if no events exist.
func (*Repository) GetSnapshot ¶
func (repo *Repository) GetSnapshot( ctx context.Context, entityID evt.EntityID, ) (*evt.SerializedSnapshot, error)
GetSnapshot gets the latest Snapshot for an Entity instance. Snapshots are stored inline in the event-log table at sk=0.
func (*Repository) PutSnapshot ¶
func (repo *Repository) PutSnapshot( ctx context.Context, entityType evt.EntityType, entityID evt.EntityID, payload []byte, snapshotSequence evt.EventSequence, eventSequence evt.EventSequence, ) error
PutSnapshot writes a snapshot directly using PutItem (no transaction). Snapshots are stored inline in the event-log table at sk=0. This is intended for use by projectors and background processes that need to write snapshots outside of the transactional command handler flow.
The write is monotonic in eventSeq: it never overwrites an existing snapshot with one whose EventSequence is lower. The sk=0 snapshot is the durable floor for both snapshot-aware load and CompactBelow — if a stale/background writer could regress it below already-compacted events, a later load would seed from the older snapshot and query events that compaction has deleted, rebuilding with missing history. A regressing write is treated as a no-op (the existing, newer snapshot already covers it), so background catch-up writers do not see spurious errors.
func (*Repository) RebuildProjectionsByQuery ¶ added in v0.5.0
func (repo *Repository) RebuildProjectionsByQuery( ctx context.Context, opts StreamByQueryOptions, applyEvent func(context.Context, evt.SerializedEvent, evt.Entity) (evt.Entity, error), cfg evt.RebuildConfig, ) (*evt.RebuildResult, error)
RebuildProjectionsByQuery rebuilds with per-entity queries and bounded in-flight aggregates. HeadSource can enumerate IDs from a heads registry.
opts.EntityType defaults to cfg.EntityType; if both are set, they must match. This path rejects SeedEntity because it cannot safely rebuild compacted streams. Use evt.RebuildProjections for snapshot-seeded replay.
func (*Repository) StreamAllEvents ¶
func (repo *Repository) StreamAllEvents( ctx context.Context, filter evt.StreamFilter, ) <-chan result.Result[[]evt.SerializedEvent]
StreamAllEvents implements the backend-neutral evt.Repository contract by translating the StreamFilter into a DynamoDB filter expression and delegating to StreamAllEventsByExpression.
func (*Repository) StreamAllEventsByExpression ¶ added in v0.6.0
func (repo *Repository) StreamAllEventsByExpression( ctx context.Context, expr *expression.Expression, ) <-chan result.Result[[]evt.SerializedEvent]
StreamAllEventsByExpression scans all Events in the table, returning a channel for results and errors.
Events are emitted page by page as they are read; the channel is a true stream and does not buffer the whole table. When the repository is configured with WithScanSegments, the scan runs in parallel across segments and pages from different segments interleave on the channel in no particular order.
func (*Repository) StreamEntities ¶
func (repo *Repository) StreamEntities( ctx context.Context, filter evt.StreamFilter, applyEvent func(context.Context, evt.SerializedEvent, evt.Entity) (evt.Entity, error), ) <-chan result.Result[evt.Entity]
StreamEntities implements the backend-neutral evt.Repository contract by translating the StreamFilter into a DynamoDB filter expression and delegating to StreamEntitiesByExpression.
func (*Repository) StreamEntitiesByExpression ¶ added in v0.6.0
func (repo *Repository) StreamEntitiesByExpression( ctx context.Context, expr *expression.Expression, applyEvent func(context.Context, evt.SerializedEvent, evt.Entity) (evt.Entity, error), ) <-chan result.Result[evt.Entity]
StreamEntitiesByExpression streams Events from the table, optionally filtered by a raw DynamoDB expression, and yields each completed Entity after all of its Events have been applied in sequence order.
A DynamoDB Scan does not guarantee that the Events for a given entity arrive contiguously or in sort-key order — and with parallel segmented scans they interleave freely — so this cannot reconstitute entities in a single streaming pass. Instead it groups every matched Event by entity ID, then, once the scan completes, sorts each entity's Events by sequence and applies them in order before yielding the entity.
As a result it buffers all matched Events in memory for the duration of the scan. It is intended for rebuild/diagnostic flows (see evt.RebuildProjections), not hot read paths. For large tables, pair it with WithScanSegments to parallelize the read.
func (*Repository) StreamEntitiesByQuery ¶ added in v0.2.0
func (repo *Repository) StreamEntitiesByQuery( ctx context.Context, opts StreamByQueryOptions, applyEvent func(context.Context, evt.SerializedEvent, evt.Entity) (evt.Entity, error), ) <-chan result.Result[evt.Entity]
StreamEntitiesByQuery reconstitutes entities with bounded memory by first enumerating the distinct entity IDs in the event log, then querying each entity's partition (which returns its events in sort-key order) and folding them with applyEvent.
Unlike StreamEntities — which scans the whole table and buffers every matched event so it can regroup the unordered scan output — this path queries one partition at a time and emits each entity as soon as it is rebuilt. Memory is bounded to the set of distinct entity IDs collected during enumeration plus up to Workers in-flight aggregates, rather than the entire event log. It is the preferred source for RebuildProjectionsFromStream on large tables.
Enumeration still uses a table Scan. Its ProjectionExpression reduces returned data, not read capacity, which DynamoDB charges by item size. Set opts.HeadSource to enumerate IDs from a heads registry without scanning the event log.
func (*Repository) StreamEntitiesFromSnapshots ¶ added in v0.2.0
func (repo *Repository) StreamEntitiesFromSnapshots( ctx context.Context, entityType evt.EntityType, seedEntity evt.SnapshotSeeder, applyEvent func(context.Context, evt.SerializedEvent, evt.Entity) (evt.Entity, error), ) <-chan result.Result[evt.Entity]
StreamEntitiesFromSnapshots reconstitutes entities with bounded memory by enumerating the distinct entity IDs in the event log and, for each, loading it from its latest durable snapshot (sk=0) plus only the events recorded after that snapshot — the same snapshot-aware path as snapshots.Store.LoadEntity, applied per partition during a rebuild. Entities without a snapshot fall back to full replay from sequence 1. This is the rebuild path that stays correct after CompactBelow has removed events below a snapshot.
It implements evt.SnapshotStreamer using the default scan-and-dedup enumeration (no schema change). For constant-memory enumeration from a heads registry, call StreamEntitiesFromSnapshotsWithOptions with StreamFromSnapshotsOptions.HeadSource set.
func (*Repository) StreamEntitiesFromSnapshotsWithOptions ¶ added in v0.5.0
func (repo *Repository) StreamEntitiesFromSnapshotsWithOptions( ctx context.Context, opts StreamFromSnapshotsOptions, seedEntity evt.SnapshotSeeder, applyEvent func(context.Context, evt.SerializedEvent, evt.Entity) (evt.Entity, error), ) <-chan result.Result[evt.Entity]
StreamEntitiesFromSnapshotsWithOptions is the configurable form of StreamEntitiesFromSnapshots. It reconstitutes each entity from its latest durable snapshot plus only the post-snapshot events, falling back to full replay for entities with no snapshot.
Like StreamEntitiesByQuery, per-entity ordering comes from partition Queries (sk-ascending), not from scan order, and the two paths share enumeration (produceEntityIDs): with opts.HeadSource set, entity IDs stream from the heads registry with no dedup set (constant memory regardless of entity count); otherwise they are collected from a key-only event-log scan first, where a failed (partial) enumeration is fatal and emits no entities so a rebuild never silently covers a subset. Memory is otherwise bounded to a small worker pool of in-flight aggregates.
func (*Repository) StreamEntityHeads ¶ added in v0.4.0
func (repo *Repository) StreamEntityHeads( ctx context.Context, entityType evt.EntityType, ) (map[evt.EntityID]evt.EventSequence, error)
StreamEntityHeads implements evt.EntityHeadStreamer by scanning the event log directly. It folds the log down to each entity's head: the larger of the highest event sort key (sk >= 1) and the snapshot's recorded eventSeq (the sk=0 row), so a stream whose early events were compacted away still reports the sequence its snapshot covers. It is the dependency-free fallback (and the backfill source for a HeadStore): correct, but it reads the whole event log every call — DynamoDB bills a Scan by item size regardless of projection. Prefer a HeadStore, backed by a small heads table maintained by the heads projector, for the steady-state change-detection path.
func (*Repository) WithClock ¶ added in v0.3.0
func (repo *Repository) WithClock(now func() time.Time) *Repository
WithClock returns a shallow copy of the repository using the given clock to compute TTL expiry timestamps. Intended for tests; production callers leave it unset to use time.Now.
func (*Repository) WithConsistentRead ¶
func (repo *Repository) WithConsistentRead(consistent bool) *Repository
WithConsistentRead returns a shallow copy of the repository with the consistent read setting updated. When false, reads use eventually consistent reads (half the RCU cost). The original repository is not modified, so it is safe to derive both a strong-consistency and eventual-consistency variant from the same base repository.
func (*Repository) WithLogger ¶
func (repo *Repository) WithLogger(logger *slog.Logger) *Repository
WithLogger returns a shallow copy of the repository with the logger updated.
func (*Repository) WithRetention ¶ added in v0.3.0
func (repo *Repository) WithRetention(retention Retention) *Repository
WithRetention returns a shallow copy of the repository that stamps a DynamoDB `ttl` attribute on committed events and inline snapshots whose entity type appears in the policy. Entity types absent from the policy are written without a `ttl` and are never auto-expired. The original repository is unchanged. See Retention for the safety constraint.
func (*Repository) WithScanSegments ¶ added in v0.2.0
func (repo *Repository) WithScanSegments(n int) *Repository
WithScanSegments returns a shallow copy of the repository configured to run table-wide reads (StreamAllEvents and, transitively, StreamEntities/RebuildProjections) as a DynamoDB parallel Scan split across n segments. n <= 1 keeps the default single sequential scan.
Parallel scans trade higher read throughput (and consumed capacity) for faster table sweeps. Pick n based on table size and provisioned/burst capacity; the original repository is unchanged so hot-path readers can keep a non-segmented copy.
type Retention ¶ added in v0.3.0
type Retention map[evt.EntityType]time.Duration
Retention maps entity types to TTL durations for events and inline snapshots. Other rows have no ttl attribute; DynamoDB TTL must be enabled on `ttl`.
Use it only for terminal, short-lived streams that are never rebuilt. TTL deletes items independently and can leave undetectable prefix gaps. For retained history, use durable snapshots and Compactor.CompactBelow.
type Snapshot ¶
type Snapshot struct {
PK evt.EntityID `json:"pk"` // the entity id is the partition key
SK evt.EventSequence `json:"sk"` // always 0 for inline snapshots
Sequence evt.EventSequence `json:"seq"` // the snapshot sequence number
EventSequence evt.EventSequence `json:"eventSeq"` // the last event sequence at the time of snapshotting
EntityType evt.EntityType `json:"entityType"` // the entity type
Payload string `json:"payload"` // JSON-serialized entity state
// TTL is the optional DynamoDB time-to-live expiry (Unix epoch seconds), written only for entity
// types covered by a Repository retention policy so an inline snapshot never outlives the events it
// summarizes. The omitempty tag drops it for un-policed types. See Repository.WithRetention.
TTL int64 `json:"ttl,omitempty"`
}
Snapshot is a struct to serialize a Snapshot for DynamoDB. Snapshots are stored inline in the event-log table at sk=0 for each entity.
type StreamByQueryOptions ¶ added in v0.2.0
type StreamByQueryOptions struct {
// EntityType, if set, restricts the rebuild to entities of this type. Enumeration filters the
// key-only scan by entityType so non-matching partitions are never queried.
EntityType evt.EntityType
// Workers is the number of entity partitions queried concurrently. Values < 1 are treated as 1
// (sequential). Each in-flight worker holds one entity's events in memory.
Workers int
// Skip, if non-nil, is consulted for each enumerated entity ID before its partition is queried.
// Returning true skips that entity. Use it to resume an interrupted run or to rebuild a subset
// (rebuilds are idempotent, so re-running from scratch is always safe; Skip just avoids redoing
// finished work).
Skip func(evt.EntityID) bool
// HeadSource, if set, enumerates entity IDs from a heads registry (one row per entity) instead
// of the default key-only event-log scan. It streams unique IDs directly to workers without a
// deduplication set. Populate the heads table with its projector and HeadStore.Backfill. The event
// log still supplies each entity's events.
//
// The default path collects every ID before emitting an entity. HeadSource emits while enumerating,
// so an enumeration error can follow emitted entities. Rebuilds are idempotent, so resume with Skip
// or rerun the rebuild.
HeadSource evt.EntityHeadVisitor
}
StreamByQueryOptions configures StreamEntitiesByQuery.
type StreamFromSnapshotsOptions ¶ added in v0.5.0
type StreamFromSnapshotsOptions struct {
// EntityType, if set, restricts the rebuild to entities of this type.
EntityType evt.EntityType
// Workers is the number of entity partitions reconstructed concurrently. Values < 1 default to
// snapshotStreamWorkers. Each in-flight worker holds one entity's post-snapshot events.
Workers int
// Skip, if non-nil, is consulted for each enumerated entity ID before it is reconstructed.
// Returning true skips that entity. Use it to resume an interrupted run or to rebuild a subset
// (rebuilds are idempotent, so re-running from scratch is always safe; Skip just avoids redoing
// finished work).
Skip func(evt.EntityID) bool
// HeadSource, if set, enumerates entity IDs from a heads registry (one row per entity) instead
// of the default key-only event-log scan. It streams unique IDs directly to workers without a
// deduplication set. Heads include a compacted stream's snapshot floor, so the registry is a valid
// ID source for snapshot-seeded rebuilds. Populate the heads table with its projector and
// HeadStore.Backfill. The event log still supplies each entity's events.
//
// The default path collects every ID before emitting an entity. HeadSource emits while enumerating,
// so an enumeration error can follow emitted entities. Rebuilds are idempotent, so resume with Skip
// or rerun the rebuild.
HeadSource evt.EntityHeadVisitor
}
StreamFromSnapshotsOptions configures StreamEntitiesFromSnapshotsWithOptions. It mirrors StreamByQueryOptions for the snapshot-aware rebuild path.
type Transaction ¶
type Transaction = []TransactionGroup
A Transaction is a collection of operations organized into TransactionGroups that should be executed together. If any one operation fails, they should all fail.
type TransactionGroup ¶
type TransactionGroup interface {
ToWriteItems() []types.TransactWriteItem
MergeDynamo(with TransactionGroup) (TransactionGroup, error)
// For compliance with the e.Transaction interface
TransactionType() evt.TransactionType
StorageType() evt.StorageType
Len() int
HandleError(err error, i int) error
Merge(with evt.TransactionGroup) (evt.TransactionGroup, error)
}
A TransactionGroup is a set of related operations that should be included in a Transaction
type View ¶
type View struct {
PK string `json:"pk"`
SK string `json:"sk"`
EntityID evt.EntityID `json:"entityID"`
EntityType evt.EntityType `json:"entityType"`
Payload string `json:"payload"`
TTL int64 `json:"ttl,omitempty"`
}
View is the Dynamo serialization for a materialized view.
type ViewProjector ¶
type ViewProjector struct {
// contains filtered or unexported fields
}
ViewProjector simply projects an Entity to a DynamoDB view by ID.
type ViewPutGroup ¶
type ViewPutGroup struct {
// contains filtered or unexported fields
}
ViewPutGroup batches DynamoDB put operations for entity views.
func NewViewPutGroup ¶
func NewViewPutGroup(tableName string, items []types.TransactWriteItem) *ViewPutGroup
NewViewPutGroup creates a new view put group for the given table name.
func (*ViewPutGroup) HandleError ¶
func (g *ViewPutGroup) HandleError(err error, _ int) error
HandleError returns the original error for Dynamo transactional failures.
func (*ViewPutGroup) Len ¶
func (g *ViewPutGroup) Len() int
Len returns the number of write operations in the group.
func (*ViewPutGroup) Merge ¶
func (g *ViewPutGroup) Merge(with evt.TransactionGroup) (evt.TransactionGroup, error)
Merge implements the generic evt.TransactionGroup interface.
func (*ViewPutGroup) MergeDynamo ¶
func (g *ViewPutGroup) MergeDynamo(with TransactionGroup) (TransactionGroup, error)
MergeDynamo merges two Dynamo-backed transaction groups.
func (*ViewPutGroup) StorageType ¶
func (g *ViewPutGroup) StorageType() evt.StorageType
StorageType identifies the backing store.
func (*ViewPutGroup) ToWriteItems ¶
func (g *ViewPutGroup) ToWriteItems() []types.TransactWriteItem
ToWriteItems exposes the DynamoDB transactions for this group.
func (*ViewPutGroup) TransactionType ¶
func (g *ViewPutGroup) TransactionType() evt.TransactionType
TransactionType identifies this transaction group.
type ViewRepository ¶
type ViewRepository struct {
ViewsTable string
// contains filtered or unexported fields
}
ViewRepository provides Dynamo-backed views.
func (*ViewRepository) BatchGetViews ¶
func (repo *ViewRepository) BatchGetViews(ctx context.Context, pks []string) ([]*evt.SerializedView, error)
BatchGetViews retrieves multiple views by partition key using the default sort key "VIEW". Keys are de-duplicated, requested via BatchGetItem in chunks of maxBatchGetItems, and UnprocessedKeys are retried with backoff. Only views that exist are returned; missing keys are omitted and order is not guaranteed, so callers should index the result by PK or by a payload identifier. Reads are strongly consistent to match GetView. Use BatchGetViewsAtSK to batch rows stored at a non-default sort key.
func (*ViewRepository) BatchGetViewsAtSK ¶
func (repo *ViewRepository) BatchGetViewsAtSK(ctx context.Context, pks []string, sk string) ([]*evt.SerializedView, error)
BatchGetViewsAtSK retrieves multiple views by partition key at the given sort key. It shares BatchGetViews' de-duplication, chunking, retry, and strong-read semantics; only the sort key differs. Callers reading rows that live at a custom sort key batch them through here instead of the DefaultViewSK shorthand.
func (*ViewRepository) DeleteView ¶
DeleteView removes a view row by primary key. If SK is empty, it defaults to "VIEW".
func (*ViewRepository) GetView ¶
func (repo *ViewRepository) GetView(ctx context.Context, pk string) (*evt.SerializedView, error)
GetView retrieves a view by PK using the default sort key "VIEW". For views with custom sort keys, use ListViewsByPK instead.
func (*ViewRepository) ListViewsByEntityType ¶
func (repo *ViewRepository) ListViewsByEntityType(ctx context.Context, entityType evt.EntityType) ([]*evt.SerializedView, error)
ListViewsByEntityType queries the entity views table via a GSI for the provided entity type.
It buffers the full result set; for large entity types use ListViewsByEntityTypePaged or ListViewsByEntityTypeEach instead.
func (*ViewRepository) ListViewsByEntityTypeEach ¶ added in v0.2.0
func (repo *ViewRepository) ListViewsByEntityTypeEach(ctx context.Context, entityType evt.EntityType, fn func(*evt.SerializedView) error) error
ListViewsByEntityTypeEach streams views for the provided entity type via the entityType GSI, invoking fn per view without buffering the whole result set.
func (*ViewRepository) ListViewsByEntityTypePaged ¶
func (repo *ViewRepository) ListViewsByEntityTypePaged(ctx context.Context, entityType evt.EntityType, limit int, cursor string) (*evt.PagedResult, error)
ListViewsByEntityTypePaged queries by entity type with cursor-based pagination.
func (*ViewRepository) ListViewsByPK ¶
func (repo *ViewRepository) ListViewsByPK(ctx context.Context, pk string) ([]*evt.SerializedView, error)
ListViewsByPK queries all views with the given partition key. Used for composite key tables where pk identifies a collection (e.g., USER#<id>#teams).
It buffers the full result set; for partition keys with many rows use ListViewsByPKPaged or ListViewsByPKEach instead.
func (*ViewRepository) ListViewsByPKEach ¶ added in v0.2.0
func (repo *ViewRepository) ListViewsByPKEach(ctx context.Context, pk string, fn func(*evt.SerializedView) error) error
ListViewsByPKEach streams views for the given partition key, invoking fn per view without buffering the whole result set.
func (*ViewRepository) ListViewsByPKPaged ¶
func (repo *ViewRepository) ListViewsByPKPaged(ctx context.Context, pk string, limit int, cursor string) (*evt.PagedResult, error)
ListViewsByPKPaged queries one partition key with a bounded result page.
func (*ViewRepository) PutView ¶
func (repo *ViewRepository) PutView(ctx context.Context, view *evt.SerializedView) error
PutView writes/replaces a view in the table. If SK is empty, it defaults to "VIEW".
func (*ViewRepository) PutViews ¶
func (repo *ViewRepository) PutViews(ctx context.Context, views []*evt.SerializedView) error
PutViews writes/replaces multiple views, coalescing them into BatchWriteItem requests of up to maxBatchWriteItems each and retrying UnprocessedItems. nil views are skipped. This lets hot-path callers that emit many views per record (e.g. one record fanning out to several views) collapse N single-item writes into ceil(N/25) round trips. If SK is empty on a view, it defaults to "VIEW".