Documentation
¶
Overview ¶
Package sync implements git-friendly memory synchronization for Engram.
Instead of a single large JSON file, memories are stored as compressed JSONL chunks with a manifest index. This design:
- Avoids git merge conflicts (each sync creates a NEW chunk, never modifies old ones)
- Keeps files small (each chunk is gzipped JSONL)
- Tracks what's been imported via chunk IDs (no duplicates)
- Works for teams (multiple devs create independent chunks)
Directory structure:
.engram/ ├── manifest.json ← index of all chunks (small, mergeable) ├── chunks/ │ ├── a3f8c1d2.jsonl.gz ← chunk 1 (compressed) │ ├── b7d2e4f1.jsonl.gz ← chunk 2 │ └── ... └── engram.db ← local working DB (gitignored)
Index ¶
- Constants
- Variables
- func CaptureUpgradeSnapshotBeforeBootstrap(s *store.Store, project string) (string, *store.CloudUpgradeState, error)
- func GetUsername() string
- func ManifestSummary(m *Manifest) string
- func RollbackProject(s *store.Store, opts UpgradeRollbackOptions) (*store.CloudUpgradeState, error)
- type ChunkData
- type ChunkEntry
- type FileTransport
- type ImportResult
- type Manifest
- type SyncResult
- type Syncer
- func New(s *store.Store, syncDir string) *Syncer
- func NewCloudWithTransport(s *store.Store, transport Transport, project string) *Syncer
- func NewLocal(s *store.Store, syncDir string) *Syncer
- func NewLocalWithProject(s *store.Store, syncDir, project string) *Syncer
- func NewWithTransport(s *store.Store, transport Transport) *Syncer
- type Transport
- type UpgradeBootstrapOptions
- type UpgradeBootstrapResult
- type UpgradeDiagnosisInput
- type UpgradeDiagnosisReport
- type UpgradeLifecycleHooks
- type UpgradeRollbackOptions
Constants ¶
const ( UpgradeStatusReady = constants.UpgradeStatusReady UpgradeStatusBlocked = constants.UpgradeStatusBlocked UpgradeReasonClassReady = constants.UpgradeClassReady UpgradeReasonClassRepairable = constants.UpgradeClassRepairable UpgradeReasonClassBlocked = constants.UpgradeClassBlocked UpgradeReasonClassPolicy = constants.UpgradeClassPolicy UpgradeReasonReady = constants.UpgradeReasonReady UpgradeReasonRepairableUnenrolled = constants.UpgradeReasonRepairableUnenrolled UpgradeReasonBlockedProjectMissing = constants.UpgradeReasonBlockedProjectMissing UpgradeReasonPolicyConfig = constants.UpgradeReasonPolicyConfig UpgradeReasonPolicyForbidden = constants.UpgradeReasonPolicyForbidden )
Variables ¶
var ErrChunkNotFound = errors.New("sync: chunk not found")
Functions ¶
func CaptureUpgradeSnapshotBeforeBootstrap ¶
func CaptureUpgradeSnapshotBeforeBootstrap(s *store.Store, project string) (string, *store.CloudUpgradeState, error)
CaptureUpgradeSnapshotBeforeBootstrap persists the enrollment state needed to roll back a bootstrap attempt and rejects unsafe post-effect checkpoints.
func GetUsername ¶
func GetUsername() string
GetUsername returns the current username for chunk attribution.
func ManifestSummary ¶
ManifestSummary returns a human-readable summary of the manifest.
func RollbackProject ¶
func RollbackProject(s *store.Store, opts UpgradeRollbackOptions) (*store.CloudUpgradeState, error)
Types ¶
type ChunkData ¶
type ChunkData struct {
Sessions []store.Session `json:"sessions"`
Observations []store.Observation `json:"observations"`
Prompts []store.Prompt `json:"prompts"`
Mutations []store.SyncMutation `json:"mutations,omitempty"`
}
ChunkData is the content of a single chunk file (JSONL entries).
type ChunkEntry ¶
type ChunkEntry struct {
ID string `json:"id"` // SHA-256 hash prefix (8 chars) of content
CreatedBy string `json:"created_by"` // Username or machine identifier
CreatedAt string `json:"created_at"` // ISO timestamp
Sessions int `json:"sessions"` // Number of sessions in chunk
Memories int `json:"memories"` // Number of observations in chunk
Prompts int `json:"prompts"` // Number of prompts in chunk
}
ChunkEntry describes a single chunk in the manifest.
type FileTransport ¶
type FileTransport struct {
// contains filtered or unexported fields
}
FileTransport reads/writes chunks to the local filesystem. This encapsulates all filesystem operations that were previously inline in the Syncer methods.
func NewFileTransport ¶
func NewFileTransport(syncDir string) *FileTransport
NewFileTransport creates a FileTransport rooted at the given sync directory.
func (*FileTransport) ReadChunk ¶
func (ft *FileTransport) ReadChunk(chunkID string) ([]byte, error)
ReadChunk reads gzipped chunk data from the chunks/ subdirectory.
func (*FileTransport) ReadManifest ¶
func (ft *FileTransport) ReadManifest() (*Manifest, error)
ReadManifest reads the manifest.json from the sync directory. Returns an empty manifest (Version=1) if the file does not exist.
func (*FileTransport) WriteChunk ¶
func (ft *FileTransport) WriteChunk(chunkID string, data []byte, _ ChunkEntry) error
WriteChunk writes gzipped chunk data to the chunks/ subdirectory.
func (*FileTransport) WriteManifest ¶
func (ft *FileTransport) WriteManifest(m *Manifest) error
WriteManifest writes the manifest to manifest.json in the sync directory.
type ImportResult ¶
type ImportResult struct {
ChunksImported int `json:"chunks_imported"`
ChunksSkipped int `json:"chunks_skipped"` // Already imported
SessionsImported int `json:"sessions_imported"`
ObservationsImported int `json:"observations_imported"`
PromptsImported int `json:"prompts_imported"`
RelationsReplayed int `json:"relations_replayed"`
RelationsDeferred int `json:"relations_deferred"`
RelationsDead int `json:"relations_dead"`
}
ImportResult is returned after importing chunks.
type Manifest ¶
type Manifest struct {
Version int `json:"version"`
Chunks []ChunkEntry `json:"chunks"`
}
Manifest is the index file that lists all chunks. This is the only file git needs to diff/merge — it's small and append-only.
type SyncResult ¶
type SyncResult struct {
ChunkID string `json:"chunk_id,omitempty"`
ChunksExported int `json:"chunks_exported,omitempty"`
SessionsExported int `json:"sessions_exported"`
ObservationsExported int `json:"observations_exported"`
PromptsExported int `json:"prompts_exported"`
MutationsExported int `json:"mutations_exported"`
IsEmpty bool `json:"is_empty"` // true if nothing new to sync
}
SyncResult is returned after a sync operation.
type Syncer ¶
type Syncer struct {
// contains filtered or unexported fields
}
Syncer handles exporting and importing memory chunks.
func New ¶
New creates a Syncer with a FileTransport rooted at syncDir. This preserves the original constructor signature for backward compatibility.
func NewCloudWithTransport ¶
NewCloudWithTransport creates a cloud-mode Syncer that enforces enrollment preflight checks before any transport/network operations.
func NewLocal ¶
NewLocal is an alias for New — creates a Syncer backed by the local filesystem. Preferred in call sites where the name makes the intent clearer.
func NewLocalWithProject ¶
NewLocalWithProject creates a filesystem Syncer whose deferred replay is limited to the supplied project. An empty project preserves all-project mode.
func NewWithTransport ¶
NewWithTransport creates a Syncer with a custom Transport implementation. This is used for remote (cloud) sync where chunks travel over HTTP.
func (*Syncer) Export ¶
func (sy *Syncer) Export(createdBy string, project string) (*SyncResult, error)
Export creates a new chunk with memories not yet in any chunk. It reads the manifest to know what's already exported, then creates a new chunk with only the new data.
func (*Syncer) Import ¶
func (sy *Syncer) Import() (*ImportResult, error)
Import reads the manifest and imports any chunks not yet in the local DB.
type Transport ¶
type Transport interface {
// ReadManifest returns the manifest (chunk index).
// Returns an empty manifest if none exists yet.
ReadManifest() (*Manifest, error)
// WriteManifest persists the manifest.
WriteManifest(m *Manifest) error
// WriteChunk writes a compressed chunk to the transport.
// chunkID is the content-addressed ID (8 hex chars).
// data is the raw JSON bytes (FileTransport gzips them; remote sends as-is).
// entry contains metadata about the chunk.
WriteChunk(chunkID string, data []byte, entry ChunkEntry) error
// ReadChunk reads a compressed chunk from the transport.
// Returns the raw bytes (gzipped for FileTransport, JSON for remote).
ReadChunk(chunkID string) ([]byte, error)
}
Transport defines how chunks are read and written during sync. This is the abstraction that allows the same Syncer to work with both local filesystem (.engram/ directory) and remote cloud server.
type UpgradeBootstrapOptions ¶
type UpgradeBootstrapResult ¶
func BootstrapProject ¶
func BootstrapProject(s *store.Store, transport Transport, opts UpgradeBootstrapOptions) (*UpgradeBootstrapResult, error)
type UpgradeDiagnosisInput ¶
type UpgradeDiagnosisReport ¶
func DiagnoseCloudUpgrade ¶
func DiagnoseCloudUpgrade(input UpgradeDiagnosisInput) (UpgradeDiagnosisReport, error)
type UpgradeLifecycleHooks ¶
type UpgradeRollbackOptions ¶
type UpgradeRollbackOptions struct {
Project string
Hooks UpgradeLifecycleHooks
}