sync

package
v1.14.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 26, 2026 License: MIT Imports: 13 Imported by: 0

Documentation

Overview

Package sync implements git-friendly memory synchronization for Engram.

Instead of a single large JSON file, memories are stored as compressed JSONL chunks with a manifest index. This design:

  • Avoids git merge conflicts (each sync creates a NEW chunk, never modifies old ones)
  • Keeps files small (each chunk is gzipped JSONL)
  • Tracks what's been imported via chunk IDs (no duplicates)
  • Works for teams (multiple devs create independent chunks)

Directory structure:

.engram/
├── manifest.json          ← index of all chunks (small, mergeable)
├── chunks/
│   ├── a3f8c1d2.jsonl.gz ← chunk 1 (compressed)
│   ├── b7d2e4f1.jsonl.gz ← chunk 2
│   └── ...
└── engram.db              ← local working DB (gitignored)

Index

Constants

View Source
const (
	UpgradeStatusReady   = constants.UpgradeStatusReady
	UpgradeStatusBlocked = constants.UpgradeStatusBlocked

	UpgradeReasonClassReady      = constants.UpgradeClassReady
	UpgradeReasonClassRepairable = constants.UpgradeClassRepairable
	UpgradeReasonClassBlocked    = constants.UpgradeClassBlocked
	UpgradeReasonClassPolicy     = constants.UpgradeClassPolicy

	UpgradeReasonReady                 = constants.UpgradeReasonReady
	UpgradeReasonRepairableUnenrolled  = constants.UpgradeReasonRepairableUnenrolled
	UpgradeReasonBlockedProjectMissing = constants.UpgradeReasonBlockedProjectMissing
	UpgradeReasonPolicyConfig          = constants.UpgradeReasonPolicyConfig
	UpgradeReasonPolicyForbidden       = constants.UpgradeReasonPolicyForbidden
)

Variables

View Source
var ErrChunkNotFound = errors.New("sync: chunk not found")

Functions

func GetUsername

func GetUsername() string

GetUsername returns the current username for chunk attribution.

func ManifestSummary

func ManifestSummary(m *Manifest) string

ManifestSummary returns a human-readable summary of the manifest.

func RollbackProject added in v1.13.0

func RollbackProject(s *store.Store, opts UpgradeRollbackOptions) (*store.CloudUpgradeState, error)

Types

type ChunkData

type ChunkData struct {
	Sessions     []store.Session      `json:"sessions"`
	Observations []store.Observation  `json:"observations"`
	Prompts      []store.Prompt       `json:"prompts"`
	Mutations    []store.SyncMutation `json:"mutations,omitempty"`
}

ChunkData is the content of a single chunk file (JSONL entries).

type ChunkEntry

type ChunkEntry struct {
	ID        string `json:"id"`         // SHA-256 hash prefix (8 chars) of content
	CreatedBy string `json:"created_by"` // Username or machine identifier
	CreatedAt string `json:"created_at"` // ISO timestamp
	Sessions  int    `json:"sessions"`   // Number of sessions in chunk
	Memories  int    `json:"memories"`   // Number of observations in chunk
	Prompts   int    `json:"prompts"`    // Number of prompts in chunk
}

ChunkEntry describes a single chunk in the manifest.

type FileTransport added in v1.8.0

type FileTransport struct {
	// contains filtered or unexported fields
}

FileTransport reads/writes chunks to the local filesystem. This encapsulates all filesystem operations that were previously inline in the Syncer methods.

func NewFileTransport added in v1.8.0

func NewFileTransport(syncDir string) *FileTransport

NewFileTransport creates a FileTransport rooted at the given sync directory.

func (*FileTransport) ReadChunk added in v1.8.0

func (ft *FileTransport) ReadChunk(chunkID string) ([]byte, error)

ReadChunk reads gzipped chunk data from the chunks/ subdirectory.

func (*FileTransport) ReadManifest added in v1.8.0

func (ft *FileTransport) ReadManifest() (*Manifest, error)

ReadManifest reads the manifest.json from the sync directory. Returns an empty manifest (Version=1) if the file does not exist.

func (*FileTransport) WriteChunk added in v1.8.0

func (ft *FileTransport) WriteChunk(chunkID string, data []byte, _ ChunkEntry) error

WriteChunk writes gzipped chunk data to the chunks/ subdirectory.

func (*FileTransport) WriteManifest added in v1.8.0

func (ft *FileTransport) WriteManifest(m *Manifest) error

WriteManifest writes the manifest to manifest.json in the sync directory.

type ImportResult

type ImportResult struct {
	ChunksImported       int `json:"chunks_imported"`
	ChunksSkipped        int `json:"chunks_skipped"` // Already imported
	SessionsImported     int `json:"sessions_imported"`
	ObservationsImported int `json:"observations_imported"`
	PromptsImported      int `json:"prompts_imported"`
}

ImportResult is returned after importing chunks.

type Manifest

type Manifest struct {
	Version int          `json:"version"`
	Chunks  []ChunkEntry `json:"chunks"`
}

Manifest is the index file that lists all chunks. This is the only file git needs to diff/merge — it's small and append-only.

type SyncResult

type SyncResult struct {
	ChunkID              string `json:"chunk_id,omitempty"`
	SessionsExported     int    `json:"sessions_exported"`
	ObservationsExported int    `json:"observations_exported"`
	PromptsExported      int    `json:"prompts_exported"`
	MutationsExported    int    `json:"mutations_exported"`
	IsEmpty              bool   `json:"is_empty"` // true if nothing new to sync
}

SyncResult is returned after a sync operation.

type Syncer

type Syncer struct {
	// contains filtered or unexported fields
}

Syncer handles exporting and importing memory chunks.

func New

func New(s *store.Store, syncDir string) *Syncer

New creates a Syncer with a FileTransport rooted at syncDir. This preserves the original constructor signature for backward compatibility.

func NewCloudWithTransport added in v1.13.0

func NewCloudWithTransport(s *store.Store, transport Transport, project string) *Syncer

NewCloudWithTransport creates a cloud-mode Syncer that enforces enrollment preflight checks before any transport/network operations.

func NewLocal added in v1.8.0

func NewLocal(s *store.Store, syncDir string) *Syncer

NewLocal is an alias for New — creates a Syncer backed by the local filesystem. Preferred in call sites where the name makes the intent clearer.

func NewWithTransport added in v1.8.0

func NewWithTransport(s *store.Store, transport Transport) *Syncer

NewWithTransport creates a Syncer with a custom Transport implementation. This is used for remote (cloud) sync where chunks travel over HTTP.

func (*Syncer) Export

func (sy *Syncer) Export(createdBy string, project string) (*SyncResult, error)

Export creates a new chunk with memories not yet in any chunk. It reads the manifest to know what's already exported, then creates a new chunk with only the new data.

func (*Syncer) Import

func (sy *Syncer) Import() (*ImportResult, error)

Import reads the manifest and imports any chunks not yet in the local DB.

func (*Syncer) Status

func (sy *Syncer) Status() (localChunks int, remoteChunks int, pendingImport int, err error)

Status returns information about what would be synced.

type Transport added in v1.8.0

type Transport interface {
	// ReadManifest returns the manifest (chunk index).
	// Returns an empty manifest if none exists yet.
	ReadManifest() (*Manifest, error)

	// WriteManifest persists the manifest.
	WriteManifest(m *Manifest) error

	// WriteChunk writes a compressed chunk to the transport.
	// chunkID is the content-addressed ID (8 hex chars).
	// data is the raw JSON bytes (FileTransport gzips them; remote sends as-is).
	// entry contains metadata about the chunk.
	WriteChunk(chunkID string, data []byte, entry ChunkEntry) error

	// ReadChunk reads a compressed chunk from the transport.
	// Returns the raw bytes (gzipped for FileTransport, JSON for remote).
	ReadChunk(chunkID string) ([]byte, error)
}

Transport defines how chunks are read and written during sync. This is the abstraction that allows the same Syncer to work with both local filesystem (.engram/ directory) and remote cloud server.

type UpgradeBootstrapOptions added in v1.13.0

type UpgradeBootstrapOptions struct {
	Project   string
	CreatedBy string
}

type UpgradeBootstrapResult added in v1.13.0

type UpgradeBootstrapResult struct {
	Project string
	Stage   string
	Resumed bool
	NoOp    bool
}

func BootstrapProject added in v1.13.0

func BootstrapProject(s *store.Store, transport Transport, opts UpgradeBootstrapOptions) (*UpgradeBootstrapResult, error)

type UpgradeDiagnosisInput added in v1.13.0

type UpgradeDiagnosisInput struct {
	Project         string
	CloudConfigured bool
	ProjectEnrolled bool
	PolicyDenied    bool
}

type UpgradeDiagnosisReport added in v1.13.0

type UpgradeDiagnosisReport struct {
	Status  string
	Class   string
	Code    string
	Message string
}

func DiagnoseCloudUpgrade added in v1.13.0

func DiagnoseCloudUpgrade(input UpgradeDiagnosisInput) (UpgradeDiagnosisReport, error)

type UpgradeLifecycleHooks added in v1.13.0

type UpgradeLifecycleHooks interface {
	StopForUpgrade(project string) error
	ResumeAfterUpgrade(project string) error
}

type UpgradeRollbackOptions added in v1.13.0

type UpgradeRollbackOptions struct {
	Project string
	Hooks   UpgradeLifecycleHooks
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL