README
¶
litefs
A FUSE filesystem for replicating SQLite databases.
Development
To develop & test LiteFS, you must run an instance of Consul. To run a Consul agent in dev mode, run the following:
consul agent -dev
Consul should then be available at http://localhost:8500
Documentation
¶
Index ¶
- Constants
- Variables
- func FormatDBID(id uint32) string
- func ParseDBID(s string) (uint32, error)
- func TrimName(name string) string
- func WriteStreamFrame(w io.Writer, f StreamFrame) error
- type Client
- type DB
- func (db *DB) CommitJournal(mode JournalMode) error
- func (db *DB) CreateJournal() (*os.File, error)
- func (db *DB) ID() uint32
- func (db *DB) InWriteTx() bool
- func (db *DB) LTXDir() string
- func (db *DB) LTXPath(minTXID, maxTXID uint64) string
- func (db *DB) Name() string
- func (db *DB) Open() error
- func (db *DB) OpenLTXFile(txID uint64) (*os.File, error)
- func (db *DB) Path() string
- func (db *DB) PendingLock() *RWMutex
- func (db *DB) Pos() Pos
- func (db *DB) ReservedLock() *RWMutex
- func (db *DB) SharedLock() *RWMutex
- func (db *DB) TXID() uint64
- func (db *DB) TryApplyLTX(path string) error
- func (db *DB) WriteDatabase(f *os.File, data []byte, offset int64) error
- func (db *DB) WriteJournal(f *os.File, data []byte, offset int64) error
- type DBStreamFrame
- type FileType
- type InodeNotifier
- type JournalMode
- type LTXStreamFrame
- type Lease
- type Leaser
- type LockType
- type Pos
- type RWMutex
- type RWMutexGuard
- type RWMutexState
- type Store
- func (s *Store) Close() error
- func (s *Store) CreateDB(name string) (*DB, *os.File, error)
- func (s *Store) DB(id uint32) *DB
- func (s *Store) DBByName(name string) *DB
- func (s *Store) DBDir(id uint32) string
- func (s *Store) DBs() []*DB
- func (s *Store) ForceCreateDB(id uint32, name string) (*DB, error)
- func (s *Store) IsPrimary() bool
- func (s *Store) MarkDirty(dbID uint32)
- func (s *Store) Open() error
- func (s *Store) Path() string
- func (s *Store) PosMap() map[uint32]Pos
- func (s *Store) PrimaryURL() string
- func (s *Store) Subscribe() *Subscriber
- func (s *Store) Unsubscribe(sub *Subscriber)
- type StreamFrame
- type StreamFrameType
- type StreamReader
- type Subscriber
Constants ¶
const ( /// Magic header string that identifies a SQLite journal header. /// https://www.sqlite.org/fileformat.html#the_rollback_journal SQLITE_JOURNAL_HEADER_STRING = "\xd9\xd5\x05\xf9\x20\xa1\x63\xd7" // Location of the database size, in pages, in the main database file. SQLITE_DATABASE_SIZE_OFFSET = 28 )
const ( LockTypePending = 0x40000000 LockTypeReserved = 0x40000001 )
const ( WALHeaderSize = 32 WALFrameHeaderSize = 24 WALIndexHeaderSize = 136 )
SQLite constants
const ( PENDING_BYTE = 0x40000000 RESERVED_BYTE = (PENDING_BYTE + 1) SHARED_FIRST = (PENDING_BYTE + 2) SHARED_SIZE = 510 )
SQLite rollback journal lock constants.
const ( WAL_WRITE_LOCK = 120 WAL_CKPT_LOCK = 121 WAL_RECOVER_LOCK = 122 WAL_READ_LOCK0 = 123 WAL_READ_LOCK1 = 124 WAL_READ_LOCK2 = 125 WAL_READ_LOCK3 = 126 WAL_READ_LOCK4 = 127 )
SQLite WAL lock constants.
const ( F_OFD_GETLK = 36 F_OFD_SETLK = 37 F_OFD_SETLKW = 38 )
Open file description lock constants.
const ( JournalModeDelete = "DELETE" JournalModeTruncate = "TRUNCATE" JournalModePersist = "PERSIST" JournalModeWAL = "WAL" )
const ( FileTypeNone = FileType(iota) FileTypeDatabase FileTypeJournal FileTypeWAL FileTypeSHM )
SQLite file types.
const ( StreamFrameTypeDB = StreamFrameType(1) StreamFrameTypeLTX = StreamFrameType(2) )
const ( RWMutexStateUnlocked = iota RWMutexStateExclusive )
const PageSize = 4096
Variables ¶
var ( ErrDatabaseNotFound = fmt.Errorf("database not found") ErrDatabaseExists = fmt.Errorf("database already exists") ErrNoPrimary = errors.New("no primary") ErrPrimaryExists = errors.New("primary exists") ErrLeaseExpired = errors.New("lease expired") ErrReadOnlyReplica = fmt.Errorf("read only replica") )
LiteFS errors
Functions ¶
func FormatDBID ¶
FormatDBID formats id as a 16-character hex string.
func WriteStreamFrame ¶
func WriteStreamFrame(w io.Writer, f StreamFrame) error
WriteStreamFrame writes the stream type & frame to the writer.
Types ¶
type Client ¶
type Client interface {
// Stream starts a long-running connection to stream changes from another node.
Stream(ctx context.Context, rawurl string, posMap map[uint32]Pos) (StreamReader, error)
}
Client represents a client for connecting to other LiteFS nodes.
type DB ¶
type DB struct {
// contains filtered or unexported fields
}
DB represents a SQLite database.
func (*DB) CommitJournal ¶
func (db *DB) CommitJournal(mode JournalMode) error
CommitJournal deletes the journal file which commits or rolls back the transaction.
func (*DB) CreateJournal ¶
CreateJournal creates a new journal file on disk.
func (*DB) OpenLTXFile ¶
OpenLTXFile returns a file handle to an LTX file that contains the given TXID.
func (*DB) PendingLock ¶
func (*DB) ReservedLock ¶
func (*DB) SharedLock ¶
func (*DB) TryApplyLTX ¶
TryApplyLTX attempts to apply an LTX file to the database.
func (*DB) WriteDatabase ¶
WriteDatabase writes data to the main database file.
type DBStreamFrame ¶
DBStreamFrame represents a frame with basic database information. This is sent at the beginning of the stream and when a new database is created.
func (*DBStreamFrame) Type ¶
func (*DBStreamFrame) Type() StreamFrameType
Type returns the type of stream frame.
type InodeNotifier ¶
InodeNotifier is a callback for the store to use to invalidate the kernel page cache.
type LTXStreamFrame ¶
type LTXStreamFrame struct {
Size int64
}
func (*LTXStreamFrame) Type ¶
func (*LTXStreamFrame) Type() StreamFrameType
Type returns the type of stream frame.
type Lease ¶
type Lease interface {
RenewedAt() time.Time
TTL() time.Duration
// Renew attempts to reset the TTL on the lease.
// Returns ErrLeaseExpired if the lease has expired or was deleted.
Renew(ctx context.Context) error
// Close attempts to remove the lease from the server.
Close() error
}
Lease represents an acquired lease from a Leaser.
type Leaser ¶
type Leaser interface {
io.Closer
AdvertiseURL() string
// Acquire attempts to acquire the lease to become the primary.
Acquire(ctx context.Context) (Lease, error)
// PrimaryURL attempts to read the current primary URL.
// Returns ErrNoPrimary if no primary has the lease.
PrimaryURL(ctx context.Context) (string, error)
}
Leaser represents an API for obtaining a lease for leader election.
type LockType ¶
type LockType int
LockType represents a SQLite lock type.
func ParseLockRange ¶
ParseLockRange returns a list of SQLite locks that are within a range.
type RWMutex ¶
type RWMutex struct {
// contains filtered or unexported fields
}
RWMutex is a reader/writer mutual exclusion lock. It wraps the sync package to provide additional capabilities such as lock upgrades & downgrades. It only supports TryLock() & TryRLock() as that is what's supported by our FUSE file system.
func (*RWMutex) State ¶
func (rw *RWMutex) State() RWMutexState
State returns whether the mutex has a exclusive lock, one or more shared locks, or if the mutex is unlocked.
func (*RWMutex) TryLock ¶
func (rw *RWMutex) TryLock() *RWMutexGuard
TryLock tries to lock the mutex for writing and returns a guard if it succeeds.
func (*RWMutex) TryRLock ¶
func (rw *RWMutex) TryRLock() *RWMutexGuard
TryRLock tries to lock rw for reading and reports whether it succeeded.
type RWMutexGuard ¶
type RWMutexGuard struct {
// contains filtered or unexported fields
}
RWMutexGuard is a reference to a held lock.
func (*RWMutexGuard) CanLock ¶
func (g *RWMutexGuard) CanLock() bool
CanLock returns true if the guard can become an exclusive lock.
func (*RWMutexGuard) RLock ¶
func (g *RWMutexGuard) RLock()
RLock downgrades the lock from an exclusive lock to a shared lock. This is a no-op if the lock is already a shared lock.
func (*RWMutexGuard) TryLock ¶
func (g *RWMutexGuard) TryLock() bool
TryLock upgrades the lock from a shared lock to an exclusive lock. This is a no-op if the lock is already an exclusive lock.
func (*RWMutexGuard) Unlock ¶
func (g *RWMutexGuard) Unlock()
Unlock unlocks the underlying mutex. Guard must be discarded after Unlock().
type RWMutexState ¶
type RWMutexState int
RWMutexState represents the lock state of an RWMutexGuard.
type Store ¶
type Store struct {
// Client used to connect to other LiteFS instances.
Client Client
// Leaser manages the lease that controls leader election.
Leaser Leaser
// Callback to notify kernel of inode changes.
InodeNotifier InodeNotifier
// contains filtered or unexported fields
}
Store represents a collection of databases.
func (*Store) CreateDB ¶
CreateDB creates a new database with the given name. The returned file handle must be closed by the caller. Returns an error if a database with the same name already exists.
func (*Store) DBByName ¶
DBByName returns a database by name. Returns nil if the database does not exist.
func (*Store) ForceCreateDB ¶
ForceCreateDB creates a database with the given ID & name. This occurs when replicating from a primary server.
func (*Store) PrimaryURL ¶
PrimaryURL returns the advertising URL of the current primary.
func (*Store) Subscribe ¶
func (s *Store) Subscribe() *Subscriber
Subscribe creates a new subscriber for store changes.
func (*Store) Unsubscribe ¶
func (s *Store) Unsubscribe(sub *Subscriber)
Unsubscribe removes a subscriber from the store.
type StreamFrame ¶
type StreamFrame interface {
io.ReaderFrom
io.WriterTo
Type() StreamFrameType
}
func ReadStreamFrame ¶
func ReadStreamFrame(r io.Reader) (StreamFrame, error)
ReadStreamFrame reads a the stream type & frame from the reader.
type StreamFrameType ¶
type StreamFrameType uint32
type StreamReader ¶
type StreamReader interface {
io.ReadCloser
// NextFrame reads the next frame from the stream. After a frame is read,
// it may have a payload that can be read via Read() until io.EOF.
NextFrame() (StreamFrame, error)
}
StreamReader represents a stream of changes from a primary server.
type Subscriber ¶
type Subscriber struct {
// contains filtered or unexported fields
}
Subscriber subscribes to changes to databases in the store.
It implements a set of "dirty" databases instead of a channel of all events as clients can be slow and we don't want to cause channels to back up. It is the responsibility of the caller to determine the state changes which is usually just checking the position of the client versus the store's database.
func (*Subscriber) Close ¶
func (s *Subscriber) Close() error
Close removes the subscriber from the store.
func (*Subscriber) DirtySet ¶
func (s *Subscriber) DirtySet() map[uint32]struct{}
DirtySet returns a set of database IDs that have changed since the last call to DirtySet(). This call clears the set.
func (*Subscriber) MarkDirty ¶
func (s *Subscriber) MarkDirty(dbID uint32)
MarkDirty marks a database ID as dirty.
func (*Subscriber) NotifyCh ¶
func (s *Subscriber) NotifyCh() <-chan struct{}
NotifyCh returns a channel that receives a value when the dirty set has changed.
Source Files
Directories