surfstore

package
v1.0.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jun 30, 2024 License: MIT Imports: 29 Imported by: 0

Documentation

Index

Constants

View Source
const CONFIG_DELIMITER string = ","
View Source
const DEFAULT_BLOCK_SIZE int = 4096
View Source
const DEFAULT_META_FILENAME string = "index.db"
View Source
const EMPTYFILE_HASHVALUE string = "-1"
View Source
const FILENAME_INDEX int = 0
View Source
const FILE_INIT_VERSION int = 1
View Source
const FILE_INIT_VERSION_STR string = "1"
View Source
const HASH_DELIMITER string = " "
View Source
const HASH_LIST_INDEX int = 2
View Source
const LOAD_FROM_DIR int = 0
View Source
const LOAD_FROM_METAFILE int = 1
View Source
const META_INIT_BY_CONFIG_STR int = 2
View Source
const META_INIT_BY_FILENAME int = 0
View Source
const META_INIT_BY_PARAMS int = 1
View Source
const NON_EXIST_FILE_VERSION int = 0
View Source
const NON_EXIST_FILE_VERSION_STR string = "0"
View Source
const SURF_CLIENT string = "[Surfstore RPCClient]:"
View Source
const SURF_SERVER string = "[Surfstore Server]:"
View Source
const TOMBSTONE_HASHVALUE string = "0"
View Source
const VERSION_INDEX int = 1

Variables

View Source
var (
	ServerStatus_name = map[int32]string{
		0: "CRASHED",
		1: "FOLLOWER",
		2: "LEADER",
	}
	ServerStatus_value = map[string]int32{
		"CRASHED":  0,
		"FOLLOWER": 1,
		"LEADER":   2,
	}
)

Enum value maps for ServerStatus.

View Source
var BlockStore_ServiceDesc = grpc.ServiceDesc{
	ServiceName: "surfstore.BlockStore",
	HandlerType: (*BlockStoreServer)(nil),
	Methods: []grpc.MethodDesc{
		{
			MethodName: "GetBlock",
			Handler:    _BlockStore_GetBlock_Handler,
		},
		{
			MethodName: "PutBlock",
			Handler:    _BlockStore_PutBlock_Handler,
		},
		{
			MethodName: "MissingBlocks",
			Handler:    _BlockStore_MissingBlocks_Handler,
		},
		{
			MethodName: "GetBlockHashes",
			Handler:    _BlockStore_GetBlockHashes_Handler,
		},
	},
	Streams:  []grpc.StreamDesc{},
	Metadata: "pkg/surfstore/surfstore.proto",
}

BlockStore_ServiceDesc is the grpc.ServiceDesc for BlockStore service. It's only intended for direct use with grpc.RegisterService, and not to be introspected or modified (even as a copy)

View Source
var ErrFileVersion = errors.New("file has an outdated version")
View Source
var ErrMissingLeader = fmt.Errorf("no leader found")
View Source
var ErrNotLeader = fmt.Errorf("server is not the leader")
View Source
var ErrServerCrashed = fmt.Errorf("server is crashed")
View Source
var ErrServerCrashedUnreachable = fmt.Errorf("server is crashed or unreachable")
View Source
var File_pkg_surfstore_surfstore_proto protoreflect.FileDescriptor
View Source
var MetaStore_ServiceDesc = grpc.ServiceDesc{
	ServiceName: "surfstore.MetaStore",
	HandlerType: (*MetaStoreServer)(nil),
	Methods: []grpc.MethodDesc{
		{
			MethodName: "GetFileInfoMap",
			Handler:    _MetaStore_GetFileInfoMap_Handler,
		},
		{
			MethodName: "UpdateFile",
			Handler:    _MetaStore_UpdateFile_Handler,
		},
		{
			MethodName: "GetBlockStoreMap",
			Handler:    _MetaStore_GetBlockStoreMap_Handler,
		},
		{
			MethodName: "GetBlockStoreAddrs",
			Handler:    _MetaStore_GetBlockStoreAddrs_Handler,
		},
	},
	Streams:  []grpc.StreamDesc{},
	Metadata: "pkg/surfstore/surfstore.proto",
}

MetaStore_ServiceDesc is the grpc.ServiceDesc for MetaStore service. It's only intended for direct use with grpc.RegisterService, and not to be introspected or modified (even as a copy)

View Source
var RaftSurfstore_ServiceDesc = grpc.ServiceDesc{
	ServiceName: "surfstore.RaftSurfstore",
	HandlerType: (*RaftSurfstoreServer)(nil),
	Methods: []grpc.MethodDesc{
		{
			MethodName: "AppendEntries",
			Handler:    _RaftSurfstore_AppendEntries_Handler,
		},
		{
			MethodName: "SetLeader",
			Handler:    _RaftSurfstore_SetLeader_Handler,
		},
		{
			MethodName: "SendHeartbeat",
			Handler:    _RaftSurfstore_SendHeartbeat_Handler,
		},
		{
			MethodName: "GetFileInfoMap",
			Handler:    _RaftSurfstore_GetFileInfoMap_Handler,
		},
		{
			MethodName: "UpdateFile",
			Handler:    _RaftSurfstore_UpdateFile_Handler,
		},
		{
			MethodName: "GetBlockStoreMap",
			Handler:    _RaftSurfstore_GetBlockStoreMap_Handler,
		},
		{
			MethodName: "GetBlockStoreAddrs",
			Handler:    _RaftSurfstore_GetBlockStoreAddrs_Handler,
		},
		{
			MethodName: "MakeServerUnreachableFrom",
			Handler:    _RaftSurfstore_MakeServerUnreachableFrom_Handler,
		},
		{
			MethodName: "Crash",
			Handler:    _RaftSurfstore_Crash_Handler,
		},
		{
			MethodName: "Restore",
			Handler:    _RaftSurfstore_Restore_Handler,
		},
		{
			MethodName: "GetInternalState",
			Handler:    _RaftSurfstore_GetInternalState_Handler,
		},
	},
	Streams:  []grpc.StreamDesc{},
	Metadata: "pkg/surfstore/surfstore.proto",
}

RaftSurfstore_ServiceDesc is the grpc.ServiceDesc for RaftSurfstore service. It's only intended for direct use with grpc.RegisterService, and not to be introspected or modified (even as a copy)

Functions

func ClientSync

func ClientSync(client RPCClient)

func ConcatPath

func ConcatPath(baseDir, fileDir string) string

File Path Related

func DownloadFile

func DownloadFile(client RPCClient, fileName string, fileMetaData *FileMetaData)

func GetBlockHashBytes

func GetBlockHashBytes(blockData []byte) []byte

Hash Related

func GetBlockHashString

func GetBlockHashString(blockData []byte) string

func GetHashListForFile

func GetHashListForFile(f *os.File, blockSize int) []string

func LoadMetaFromMetaFile

func LoadMetaFromMetaFile(baseDir string) (fileMetaMap map[string]*FileMetaData, e error)

LoadMetaFromMetaFile loads the local metadata file into a file meta map. The key is the file's name and the value is the file's metadata. You can use this function to load the index.db file in this project.

func PrintMetaMap

func PrintMetaMap(metaMap map[string]*FileMetaData)

PrintMetaMap prints the contents of the metadata map. You might find this function useful for debugging.

func RegisterBlockStoreServer

func RegisterBlockStoreServer(s grpc.ServiceRegistrar, srv BlockStoreServer)

func RegisterMetaStoreServer

func RegisterMetaStoreServer(s grpc.ServiceRegistrar, srv MetaStoreServer)

func RegisterRaftSurfstoreServer

func RegisterRaftSurfstoreServer(s grpc.ServiceRegistrar, srv RaftSurfstoreServer)

func ServeRaftServer

func ServeRaftServer(s *RaftSurfstore) error

Start the Raft server

func UploadFile

func UploadFile(client RPCClient, fileName string, fileMetaData *FileMetaData) error

func WriteMetaFile

func WriteMetaFile(fileMetadataMap map[string]*FileMetaData, baseDir string) error

WriteMetaFile writes the file meta map back to local metadata file index.db

Types

type AppendEntryInput

type AppendEntryInput struct {
	Term         int64              `protobuf:"varint,1,opt,name=term,proto3" json:"term,omitempty"`
	LeaderId     int64              `protobuf:"varint,2,opt,name=leaderId,proto3" json:"leaderId,omitempty"`
	PrevLogIndex int64              `protobuf:"varint,3,opt,name=prevLogIndex,proto3" json:"prevLogIndex,omitempty"`
	PrevLogTerm  int64              `protobuf:"varint,4,opt,name=prevLogTerm,proto3" json:"prevLogTerm,omitempty"`
	Entries      []*UpdateOperation `protobuf:"bytes,5,rep,name=entries,proto3" json:"entries,omitempty"`
	LeaderCommit int64              `protobuf:"varint,6,opt,name=leaderCommit,proto3" json:"leaderCommit,omitempty"`
	// contains filtered or unexported fields
}

func (*AppendEntryInput) Descriptor deprecated

func (*AppendEntryInput) Descriptor() ([]byte, []int)

Deprecated: Use AppendEntryInput.ProtoReflect.Descriptor instead.

func (*AppendEntryInput) GetEntries

func (x *AppendEntryInput) GetEntries() []*UpdateOperation

func (*AppendEntryInput) GetLeaderCommit

func (x *AppendEntryInput) GetLeaderCommit() int64

func (*AppendEntryInput) GetLeaderId

func (x *AppendEntryInput) GetLeaderId() int64

func (*AppendEntryInput) GetPrevLogIndex

func (x *AppendEntryInput) GetPrevLogIndex() int64

func (*AppendEntryInput) GetPrevLogTerm

func (x *AppendEntryInput) GetPrevLogTerm() int64

func (*AppendEntryInput) GetTerm

func (x *AppendEntryInput) GetTerm() int64

func (*AppendEntryInput) ProtoMessage

func (*AppendEntryInput) ProtoMessage()

func (*AppendEntryInput) ProtoReflect

func (x *AppendEntryInput) ProtoReflect() protoreflect.Message

func (*AppendEntryInput) Reset

func (x *AppendEntryInput) Reset()

func (*AppendEntryInput) String

func (x *AppendEntryInput) String() string

type AppendEntryOutput

type AppendEntryOutput struct {
	ServerId     int64 `protobuf:"varint,1,opt,name=serverId,proto3" json:"serverId,omitempty"`
	Term         int64 `protobuf:"varint,2,opt,name=term,proto3" json:"term,omitempty"`
	Success      bool  `protobuf:"varint,3,opt,name=success,proto3" json:"success,omitempty"`
	MatchedIndex int64 `protobuf:"varint,4,opt,name=matchedIndex,proto3" json:"matchedIndex,omitempty"`
	// contains filtered or unexported fields
}

func (*AppendEntryOutput) Descriptor deprecated

func (*AppendEntryOutput) Descriptor() ([]byte, []int)

Deprecated: Use AppendEntryOutput.ProtoReflect.Descriptor instead.

func (*AppendEntryOutput) GetMatchedIndex

func (x *AppendEntryOutput) GetMatchedIndex() int64

func (*AppendEntryOutput) GetServerId

func (x *AppendEntryOutput) GetServerId() int64

func (*AppendEntryOutput) GetSuccess

func (x *AppendEntryOutput) GetSuccess() bool

func (*AppendEntryOutput) GetTerm

func (x *AppendEntryOutput) GetTerm() int64

func (*AppendEntryOutput) ProtoMessage

func (*AppendEntryOutput) ProtoMessage()

func (*AppendEntryOutput) ProtoReflect

func (x *AppendEntryOutput) ProtoReflect() protoreflect.Message

func (*AppendEntryOutput) Reset

func (x *AppendEntryOutput) Reset()

func (*AppendEntryOutput) String

func (x *AppendEntryOutput) String() string

type Block

type Block struct {
	BlockData []byte `protobuf:"bytes,1,opt,name=blockData,proto3" json:"blockData,omitempty"`
	BlockSize int32  `protobuf:"varint,2,opt,name=blockSize,proto3" json:"blockSize,omitempty"`
	// contains filtered or unexported fields
}

func (*Block) Descriptor deprecated

func (*Block) Descriptor() ([]byte, []int)

Deprecated: Use Block.ProtoReflect.Descriptor instead.

func (*Block) GetBlockData

func (x *Block) GetBlockData() []byte

func (*Block) GetBlockSize

func (x *Block) GetBlockSize() int32

func (*Block) ProtoMessage

func (*Block) ProtoMessage()

func (*Block) ProtoReflect

func (x *Block) ProtoReflect() protoreflect.Message

func (*Block) Reset

func (x *Block) Reset()

func (*Block) String

func (x *Block) String() string

type BlockHash

type BlockHash struct {
	Hash string `protobuf:"bytes,1,opt,name=hash,proto3" json:"hash,omitempty"`
	// contains filtered or unexported fields
}

func (*BlockHash) Descriptor deprecated

func (*BlockHash) Descriptor() ([]byte, []int)

Deprecated: Use BlockHash.ProtoReflect.Descriptor instead.

func (*BlockHash) GetHash

func (x *BlockHash) GetHash() string

func (*BlockHash) ProtoMessage

func (*BlockHash) ProtoMessage()

func (*BlockHash) ProtoReflect

func (x *BlockHash) ProtoReflect() protoreflect.Message

func (*BlockHash) Reset

func (x *BlockHash) Reset()

func (*BlockHash) String

func (x *BlockHash) String() string

type BlockHashes

type BlockHashes struct {
	Hashes []string `protobuf:"bytes,1,rep,name=hashes,proto3" json:"hashes,omitempty"`
	// contains filtered or unexported fields
}

func (*BlockHashes) Descriptor deprecated

func (*BlockHashes) Descriptor() ([]byte, []int)

Deprecated: Use BlockHashes.ProtoReflect.Descriptor instead.

func (*BlockHashes) GetHashes

func (x *BlockHashes) GetHashes() []string

func (*BlockHashes) ProtoMessage

func (*BlockHashes) ProtoMessage()

func (*BlockHashes) ProtoReflect

func (x *BlockHashes) ProtoReflect() protoreflect.Message

func (*BlockHashes) Reset

func (x *BlockHashes) Reset()

func (*BlockHashes) String

func (x *BlockHashes) String() string

type BlockStore

type BlockStore struct {
	BlockMap map[string]*Block
	UnimplementedBlockStoreServer
}

func NewBlockStore

func NewBlockStore() *BlockStore

func (*BlockStore) GetBlock

func (bs *BlockStore) GetBlock(ctx context.Context, blockHash *BlockHash) (*Block, error)

Retrieves a block indexed by hash value h

func (*BlockStore) GetBlockHashes

func (bs *BlockStore) GetBlockHashes(ctx context.Context, _ *emptypb.Empty) (*BlockHashes, error)

Returns a list containing all block hashes on this block server

func (*BlockStore) MissingBlocks

func (bs *BlockStore) MissingBlocks(ctx context.Context, blockHashesIn *BlockHashes) (*BlockHashes, error)

Given a list of hashes, returns a list containing the hashes that are NOT stored in the key-value store

func (*BlockStore) PutBlock

func (bs *BlockStore) PutBlock(ctx context.Context, block *Block) (*Success, error)

Stores block b in the key-value store, indexed by hash value h

type BlockStoreAddrs

type BlockStoreAddrs struct {
	BlockStoreAddrs []string `protobuf:"bytes,1,rep,name=blockStoreAddrs,proto3" json:"blockStoreAddrs,omitempty"`
	// contains filtered or unexported fields
}

func (*BlockStoreAddrs) Descriptor deprecated

func (*BlockStoreAddrs) Descriptor() ([]byte, []int)

Deprecated: Use BlockStoreAddrs.ProtoReflect.Descriptor instead.

func (*BlockStoreAddrs) GetBlockStoreAddrs

func (x *BlockStoreAddrs) GetBlockStoreAddrs() []string

func (*BlockStoreAddrs) ProtoMessage

func (*BlockStoreAddrs) ProtoMessage()

func (*BlockStoreAddrs) ProtoReflect

func (x *BlockStoreAddrs) ProtoReflect() protoreflect.Message

func (*BlockStoreAddrs) Reset

func (x *BlockStoreAddrs) Reset()

func (*BlockStoreAddrs) String

func (x *BlockStoreAddrs) String() string

type BlockStoreClient

type BlockStoreClient interface {
	GetBlock(ctx context.Context, in *BlockHash, opts ...grpc.CallOption) (*Block, error)
	PutBlock(ctx context.Context, in *Block, opts ...grpc.CallOption) (*Success, error)
	MissingBlocks(ctx context.Context, in *BlockHashes, opts ...grpc.CallOption) (*BlockHashes, error)
	GetBlockHashes(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*BlockHashes, error)
}

BlockStoreClient is the client API for BlockStore service.

For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.

func NewBlockStoreClient

func NewBlockStoreClient(cc grpc.ClientConnInterface) BlockStoreClient

type BlockStoreInterface

type BlockStoreInterface interface {

	// Get a block based on
	GetBlock(ctx context.Context, blockHash *BlockHash) (*Block, error)

	// Put a block
	PutBlock(ctx context.Context, block *Block) (*Success, error)

	// Given a list of hashes “in”, returns a list containing the
	// subset of in that are NOT stored in the key-value store
	MissingBlocks(ctx context.Context, blockHashesIn *BlockHashes) (*BlockHashes, error)

	// Get which blocks are on this BlockStore server
	GetBlockHashes(ctx context.Context, _ *emptypb.Empty) (*BlockHashes, error)
}

type BlockStoreMap

type BlockStoreMap struct {
	BlockStoreMap map[string]*BlockHashes `` /* 167-byte string literal not displayed */
	// contains filtered or unexported fields
}

func (*BlockStoreMap) Descriptor deprecated

func (*BlockStoreMap) Descriptor() ([]byte, []int)

Deprecated: Use BlockStoreMap.ProtoReflect.Descriptor instead.

func (*BlockStoreMap) GetBlockStoreMap

func (x *BlockStoreMap) GetBlockStoreMap() map[string]*BlockHashes

func (*BlockStoreMap) ProtoMessage

func (*BlockStoreMap) ProtoMessage()

func (*BlockStoreMap) ProtoReflect

func (x *BlockStoreMap) ProtoReflect() protoreflect.Message

func (*BlockStoreMap) Reset

func (x *BlockStoreMap) Reset()

func (*BlockStoreMap) String

func (x *BlockStoreMap) String() string

type BlockStoreServer

type BlockStoreServer interface {
	GetBlock(context.Context, *BlockHash) (*Block, error)
	PutBlock(context.Context, *Block) (*Success, error)
	MissingBlocks(context.Context, *BlockHashes) (*BlockHashes, error)
	GetBlockHashes(context.Context, *empty.Empty) (*BlockHashes, error)
	// contains filtered or unexported methods
}

BlockStoreServer is the server API for BlockStore service. All implementations must embed UnimplementedBlockStoreServer for forward compatibility

type ClientInterface

type ClientInterface interface {
	// MetaStore
	GetFileInfoMap(serverFileInfoMap *map[string]*FileMetaData) error
	UpdateFile(fileMetaData *FileMetaData) (int32, error)
	GetBlockStoreMap(blockHashesIn []string, blockStoreMap *map[string][]string) error
	GetBlockStoreAddrs(blockStoreAddrs *[]string) error

	// BlockStore
	GetBlock(blockHash string, blockStoreAddr string, block *Block) error
	PutBlock(block *Block, blockStoreAddr string) (bool, error)
	MissingBlocks(blockHashesIn []string, blockStoreAddr string, blockHashesOut *[]string) error
	GetBlockHashes(blockStoreAddr string, blockHashes *[]string) error
}

type ConsistentHashRing

type ConsistentHashRing struct {
	ServerMap map[string]string
}

func NewConsistentHashRing

func NewConsistentHashRing(serverAddrs []string) *ConsistentHashRing

func (ConsistentHashRing) GetResponsibleServer

func (c ConsistentHashRing) GetResponsibleServer(blockHash string) string

GetResponsibleServer returns the address of the BlockStore server that holds the block with the given block hash.

func (ConsistentHashRing) Hash

func (c ConsistentHashRing) Hash(addr string) string

type FileInfoMap

type FileInfoMap struct {
	FileInfoMap map[string]*FileMetaData `` /* 163-byte string literal not displayed */
	// contains filtered or unexported fields
}

func (*FileInfoMap) Descriptor deprecated

func (*FileInfoMap) Descriptor() ([]byte, []int)

Deprecated: Use FileInfoMap.ProtoReflect.Descriptor instead.

func (*FileInfoMap) GetFileInfoMap

func (x *FileInfoMap) GetFileInfoMap() map[string]*FileMetaData

func (*FileInfoMap) ProtoMessage

func (*FileInfoMap) ProtoMessage()

func (*FileInfoMap) ProtoReflect

func (x *FileInfoMap) ProtoReflect() protoreflect.Message

func (*FileInfoMap) Reset

func (x *FileInfoMap) Reset()

func (*FileInfoMap) String

func (x *FileInfoMap) String() string

type FileMetaData

type FileMetaData struct {
	Filename      string   `protobuf:"bytes,1,opt,name=filename,proto3" json:"filename,omitempty"`
	Version       int32    `protobuf:"varint,2,opt,name=version,proto3" json:"version,omitempty"`
	BlockHashList []string `protobuf:"bytes,3,rep,name=blockHashList,proto3" json:"blockHashList,omitempty"`
	// contains filtered or unexported fields
}

func (*FileMetaData) Descriptor deprecated

func (*FileMetaData) Descriptor() ([]byte, []int)

Deprecated: Use FileMetaData.ProtoReflect.Descriptor instead.

func (*FileMetaData) GetBlockHashList

func (x *FileMetaData) GetBlockHashList() []string

func (*FileMetaData) GetFilename

func (x *FileMetaData) GetFilename() string

func (*FileMetaData) GetVersion

func (x *FileMetaData) GetVersion() int32

func (*FileMetaData) ProtoMessage

func (*FileMetaData) ProtoMessage()

func (*FileMetaData) ProtoReflect

func (x *FileMetaData) ProtoReflect() protoreflect.Message

func (*FileMetaData) Reset

func (x *FileMetaData) Reset()

func (*FileMetaData) String

func (x *FileMetaData) String() string

type MetaStore

type MetaStore struct {
	FileMetaMap        map[string]*FileMetaData
	BlockStoreAddrs    []string
	ConsistentHashRing *ConsistentHashRing
	UnimplementedMetaStoreServer
}

func NewMetaStore

func NewMetaStore(blockStoreAddrs []string) *MetaStore

func (*MetaStore) GetBlockStoreAddrs

func (m *MetaStore) GetBlockStoreAddrs(ctx context.Context, _ *emptypb.Empty) (*BlockStoreAddrs, error)

GetBlockStoreAddrs returns all the BlockStore addresses.

func (*MetaStore) GetBlockStoreMap

func (m *MetaStore) GetBlockStoreMap(ctx context.Context, blockHashesIn *BlockHashes) (*BlockStoreMap, error)

GetBlockStoreMap determines the responsible block server for each block hash provided in the input and returns a mapping of block server addresses to their respective block hashes.

func (*MetaStore) GetFileInfoMap

func (m *MetaStore) GetFileInfoMap(ctx context.Context, _ *emptypb.Empty) (*FileInfoMap, error)

Returns a mapping of the files stored in the SurfStore cloud service, including the version, filename, and hashlist.

func (*MetaStore) UpdateFile

func (m *MetaStore) UpdateFile(ctx context.Context, fileMetaData *FileMetaData) (*Version, error)

Updates the FileInfo values associated with a file stored in the cloud. This method replaces the hash list for the file with the provided hash list only if the new version number is exactly one greater than the current version number. Otherwise, it returns version=-1 to the client telling them that the version they are trying to store is not right (likely too old).

type MetaStoreClient

type MetaStoreClient interface {
	GetFileInfoMap(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*FileInfoMap, error)
	UpdateFile(ctx context.Context, in *FileMetaData, opts ...grpc.CallOption) (*Version, error)
	GetBlockStoreMap(ctx context.Context, in *BlockHashes, opts ...grpc.CallOption) (*BlockStoreMap, error)
	GetBlockStoreAddrs(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*BlockStoreAddrs, error)
}

MetaStoreClient is the client API for MetaStore service.

For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.

func NewMetaStoreClient

func NewMetaStoreClient(cc grpc.ClientConnInterface) MetaStoreClient

type MetaStoreInterface

type MetaStoreInterface interface {
	// Retrieves the server's FileInfoMap
	GetFileInfoMap(ctx context.Context, _ *emptypb.Empty) (*FileInfoMap, error)

	// Update a file's fileinfo entry
	UpdateFile(ctx context.Context, fileMetaData *FileMetaData) (*Version, error)

	// Retrieve the mapping of BlockStore addresses to block hashes
	GetBlockStoreMap(ctx context.Context, blockHashesIn *BlockHashes) (*BlockStoreMap, error)

	// Retrieve all BlockStore Addresses
	GetBlockStoreAddrs(ctx context.Context, _ *emptypb.Empty) (*BlockStoreAddrs, error)
}

type MetaStoreServer

type MetaStoreServer interface {
	GetFileInfoMap(context.Context, *empty.Empty) (*FileInfoMap, error)
	UpdateFile(context.Context, *FileMetaData) (*Version, error)
	GetBlockStoreMap(context.Context, *BlockHashes) (*BlockStoreMap, error)
	GetBlockStoreAddrs(context.Context, *empty.Empty) (*BlockStoreAddrs, error)
	// contains filtered or unexported methods
}

MetaStoreServer is the server API for MetaStore service. All implementations must embed UnimplementedMetaStoreServer for forward compatibility

type RPCClient

type RPCClient struct {
	MetaStoreAddrs []string
	BaseDir        string
	BlockSize      int
}

func NewSurfstoreRPCClient

func NewSurfstoreRPCClient(addrs []string, baseDir string, blockSize int) RPCClient

Create an Surfstore RPC client

func (*RPCClient) GetBlock

func (surfClient *RPCClient) GetBlock(blockHash string, blockStoreAddr string, block *Block) error

func (*RPCClient) GetBlockHashes

func (surfClient *RPCClient) GetBlockHashes(blockStoreAddr string, blockHashes *[]string) error

func (*RPCClient) GetBlockStoreAddrs

func (surfClient *RPCClient) GetBlockStoreAddrs(blockStoreAddrs *[]string) error

func (*RPCClient) GetBlockStoreMap

func (surfClient *RPCClient) GetBlockStoreMap(blockHashesIn []string, blockStoreMap *map[string][]string) error

func (*RPCClient) GetFileInfoMap

func (surfClient *RPCClient) GetFileInfoMap(serverFileInfoMap *map[string]*FileMetaData) error

func (*RPCClient) MissingBlocks

func (surfClient *RPCClient) MissingBlocks(blockHashesIn []string, blockStoreAddr string, blockHashesOut *[]string) error

func (*RPCClient) PutBlock

func (surfClient *RPCClient) PutBlock(block *Block, blockStoreAddr string) (bool, error)

func (*RPCClient) UpdateFile

func (surfClient *RPCClient) UpdateFile(fileMetaData *FileMetaData) (int32, error)

type RaftConfig

type RaftConfig struct {
	RaftAddrs  []string
	BlockAddrs []string
}

Raft configuration

func LoadRaftConfigFile

func LoadRaftConfigFile(filename string) (cfg RaftConfig)

Load the Raft configuration file

type RaftInterface

type RaftInterface interface {
	AppendEntries(ctx context.Context, input *AppendEntryInput) (*AppendEntryOutput, error)
	SetLeader(ctx context.Context, _ *emptypb.Empty) (*Success, error)
	SendHeartbeat(ctx context.Context, _ *emptypb.Empty) (*Success, error)
}

type RaftInternalState

type RaftInternalState struct {
	Status      ServerStatus       `protobuf:"varint,1,opt,name=status,proto3,enum=surfstore.ServerStatus" json:"status,omitempty"`
	Term        int64              `protobuf:"varint,2,opt,name=term,proto3" json:"term,omitempty"`
	CommitIndex int64              `protobuf:"varint,3,opt,name=commitIndex,proto3" json:"commitIndex,omitempty"`
	Log         []*UpdateOperation `protobuf:"bytes,4,rep,name=log,proto3" json:"log,omitempty"`
	MetaMap     *FileInfoMap       `protobuf:"bytes,5,opt,name=metaMap,proto3" json:"metaMap,omitempty"`
	// contains filtered or unexported fields
}

func (*RaftInternalState) Descriptor deprecated

func (*RaftInternalState) Descriptor() ([]byte, []int)

Deprecated: Use RaftInternalState.ProtoReflect.Descriptor instead.

func (*RaftInternalState) GetCommitIndex

func (x *RaftInternalState) GetCommitIndex() int64

func (*RaftInternalState) GetLog

func (x *RaftInternalState) GetLog() []*UpdateOperation

func (*RaftInternalState) GetMetaMap

func (x *RaftInternalState) GetMetaMap() *FileInfoMap

func (*RaftInternalState) GetStatus

func (x *RaftInternalState) GetStatus() ServerStatus

func (*RaftInternalState) GetTerm

func (x *RaftInternalState) GetTerm() int64

func (*RaftInternalState) ProtoMessage

func (*RaftInternalState) ProtoMessage()

func (*RaftInternalState) ProtoReflect

func (x *RaftInternalState) ProtoReflect() protoreflect.Message

func (*RaftInternalState) Reset

func (x *RaftInternalState) Reset()

func (*RaftInternalState) String

func (x *RaftInternalState) String() string

type RaftSurfstore

type RaftSurfstore struct {
	UnimplementedRaftSurfstoreServer
	// contains filtered or unexported fields
}

func NewRaftServer

func NewRaftServer(id int64, config RaftConfig) (*RaftSurfstore, error)

Initialize a new Raft server

func (*RaftSurfstore) AppendEntries

func (s *RaftSurfstore) AppendEntries(ctx context.Context, input *AppendEntryInput) (*AppendEntryOutput, error)

func (*RaftSurfstore) Crash

func (s *RaftSurfstore) Crash(ctx context.Context, _ *emptypb.Empty) (*Success, error)

func (*RaftSurfstore) GetBlockStoreAddrs

func (s *RaftSurfstore) GetBlockStoreAddrs(ctx context.Context, empty *emptypb.Empty) (*BlockStoreAddrs, error)

func (*RaftSurfstore) GetBlockStoreMap

func (s *RaftSurfstore) GetBlockStoreMap(ctx context.Context, hashes *BlockHashes) (*BlockStoreMap, error)

func (*RaftSurfstore) GetFileInfoMap

func (s *RaftSurfstore) GetFileInfoMap(ctx context.Context, empty *emptypb.Empty) (*FileInfoMap, error)

func (*RaftSurfstore) GetInternalState

func (s *RaftSurfstore) GetInternalState(ctx context.Context, empty *emptypb.Empty) (*RaftInternalState, error)

func (*RaftSurfstore) MakeServerUnreachableFrom

func (s *RaftSurfstore) MakeServerUnreachableFrom(ctx context.Context, servers *UnreachableFromServers) (*Success, error)

func (*RaftSurfstore) Restore

func (s *RaftSurfstore) Restore(ctx context.Context, _ *emptypb.Empty) (*Success, error)

func (*RaftSurfstore) SendHeartbeat

func (s *RaftSurfstore) SendHeartbeat(ctx context.Context, _ *emptypb.Empty) (*Success, error)

Sends a round of AppendEntries to all other nodes. The leader will attempt to replicate logs to all other nodes when this is called. It can be called even when there are no entries to replicate.

func (*RaftSurfstore) SetLeader

func (s *RaftSurfstore) SetLeader(ctx context.Context, _ *emptypb.Empty) (*Success, error)

The SetLeader() function should emulate an election, so after calling it on a node it should set all the state as if that node had just won an election.

func (*RaftSurfstore) UpdateFile

func (s *RaftSurfstore) UpdateFile(ctx context.Context, filemeta *FileMetaData) (*Version, error)

type RaftSurfstoreClient

type RaftSurfstoreClient interface {
	// Raft
	AppendEntries(ctx context.Context, in *AppendEntryInput, opts ...grpc.CallOption) (*AppendEntryOutput, error)
	SetLeader(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*Success, error)
	SendHeartbeat(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*Success, error)
	// Metastore
	GetFileInfoMap(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*FileInfoMap, error)
	UpdateFile(ctx context.Context, in *FileMetaData, opts ...grpc.CallOption) (*Version, error)
	GetBlockStoreMap(ctx context.Context, in *BlockHashes, opts ...grpc.CallOption) (*BlockStoreMap, error)
	GetBlockStoreAddrs(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*BlockStoreAddrs, error)
	// Testing Interface
	MakeServerUnreachableFrom(ctx context.Context, in *UnreachableFromServers, opts ...grpc.CallOption) (*Success, error)
	Crash(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*Success, error)
	Restore(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*Success, error)
	GetInternalState(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*RaftInternalState, error)
}

RaftSurfstoreClient is the client API for RaftSurfstore service.

For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.

type RaftSurfstoreInterface

type RaftSurfstoreInterface interface {
	MetaStoreInterface
	RaftInterface
	RaftTestingInterface
}

type RaftSurfstoreServer

type RaftSurfstoreServer interface {
	// Raft
	AppendEntries(context.Context, *AppendEntryInput) (*AppendEntryOutput, error)
	SetLeader(context.Context, *empty.Empty) (*Success, error)
	SendHeartbeat(context.Context, *empty.Empty) (*Success, error)
	// Metastore
	GetFileInfoMap(context.Context, *empty.Empty) (*FileInfoMap, error)
	UpdateFile(context.Context, *FileMetaData) (*Version, error)
	GetBlockStoreMap(context.Context, *BlockHashes) (*BlockStoreMap, error)
	GetBlockStoreAddrs(context.Context, *empty.Empty) (*BlockStoreAddrs, error)
	// Testing Interface
	MakeServerUnreachableFrom(context.Context, *UnreachableFromServers) (*Success, error)
	Crash(context.Context, *empty.Empty) (*Success, error)
	Restore(context.Context, *empty.Empty) (*Success, error)
	GetInternalState(context.Context, *empty.Empty) (*RaftInternalState, error)
	// contains filtered or unexported methods
}

RaftSurfstoreServer is the server API for RaftSurfstore service. All implementations must embed UnimplementedRaftSurfstoreServer for forward compatibility

type RaftTestingInterface

type RaftTestingInterface interface {
	GetInternalState(ctx context.Context, _ *emptypb.Empty) (*RaftInternalState, error)
	Crash(ctx context.Context, _ *emptypb.Empty) (*Success, error)
	MakeServerUnreachableFrom(ctx context.Context, servers *UnreachableFromServers) (*Success, error)
	Restore(ctx context.Context, _ *emptypb.Empty) (*Success, error)
}

type ServerStatus

type ServerStatus int32
const (
	ServerStatus_CRASHED  ServerStatus = 0
	ServerStatus_FOLLOWER ServerStatus = 1
	ServerStatus_LEADER   ServerStatus = 2
)

func (ServerStatus) Descriptor

func (ServerStatus) Enum

func (x ServerStatus) Enum() *ServerStatus

func (ServerStatus) EnumDescriptor deprecated

func (ServerStatus) EnumDescriptor() ([]byte, []int)

Deprecated: Use ServerStatus.Descriptor instead.

func (ServerStatus) Number

func (ServerStatus) String

func (x ServerStatus) String() string

func (ServerStatus) Type

type Success

type Success struct {
	Flag bool `protobuf:"varint,1,opt,name=flag,proto3" json:"flag,omitempty"`
	// contains filtered or unexported fields
}

func (*Success) Descriptor deprecated

func (*Success) Descriptor() ([]byte, []int)

Deprecated: Use Success.ProtoReflect.Descriptor instead.

func (*Success) GetFlag

func (x *Success) GetFlag() bool

func (*Success) ProtoMessage

func (*Success) ProtoMessage()

func (*Success) ProtoReflect

func (x *Success) ProtoReflect() protoreflect.Message

func (*Success) Reset

func (x *Success) Reset()

func (*Success) String

func (x *Success) String() string

type UnimplementedBlockStoreServer

type UnimplementedBlockStoreServer struct {
}

UnimplementedBlockStoreServer must be embedded to have forward compatible implementations.

func (UnimplementedBlockStoreServer) GetBlock

func (UnimplementedBlockStoreServer) GetBlockHashes

func (UnimplementedBlockStoreServer) MissingBlocks

func (UnimplementedBlockStoreServer) PutBlock

type UnimplementedMetaStoreServer

type UnimplementedMetaStoreServer struct {
}

UnimplementedMetaStoreServer must be embedded to have forward compatible implementations.

func (UnimplementedMetaStoreServer) GetBlockStoreAddrs

func (UnimplementedMetaStoreServer) GetBlockStoreMap

func (UnimplementedMetaStoreServer) GetFileInfoMap

func (UnimplementedMetaStoreServer) UpdateFile

type UnimplementedRaftSurfstoreServer

type UnimplementedRaftSurfstoreServer struct {
}

UnimplementedRaftSurfstoreServer must be embedded to have forward compatible implementations.

func (UnimplementedRaftSurfstoreServer) AppendEntries

func (UnimplementedRaftSurfstoreServer) Crash

func (UnimplementedRaftSurfstoreServer) GetBlockStoreAddrs

func (UnimplementedRaftSurfstoreServer) GetBlockStoreMap

func (UnimplementedRaftSurfstoreServer) GetFileInfoMap

func (UnimplementedRaftSurfstoreServer) GetInternalState

func (UnimplementedRaftSurfstoreServer) MakeServerUnreachableFrom

func (UnimplementedRaftSurfstoreServer) Restore

func (UnimplementedRaftSurfstoreServer) SendHeartbeat

func (UnimplementedRaftSurfstoreServer) SetLeader

func (UnimplementedRaftSurfstoreServer) UpdateFile

type UnreachableFromServers

type UnreachableFromServers struct {
	ServerIds []int64 `protobuf:"varint,1,rep,packed,name=serverIds,proto3" json:"serverIds,omitempty"`
	// contains filtered or unexported fields
}

func (*UnreachableFromServers) Descriptor deprecated

func (*UnreachableFromServers) Descriptor() ([]byte, []int)

Deprecated: Use UnreachableFromServers.ProtoReflect.Descriptor instead.

func (*UnreachableFromServers) GetServerIds

func (x *UnreachableFromServers) GetServerIds() []int64

func (*UnreachableFromServers) ProtoMessage

func (*UnreachableFromServers) ProtoMessage()

func (*UnreachableFromServers) ProtoReflect

func (x *UnreachableFromServers) ProtoReflect() protoreflect.Message

func (*UnreachableFromServers) Reset

func (x *UnreachableFromServers) Reset()

func (*UnreachableFromServers) String

func (x *UnreachableFromServers) String() string

type UnsafeBlockStoreServer

type UnsafeBlockStoreServer interface {
	// contains filtered or unexported methods
}

UnsafeBlockStoreServer may be embedded to opt out of forward compatibility for this service. Use of this interface is not recommended, as added methods to BlockStoreServer will result in compilation errors.

type UnsafeMetaStoreServer

type UnsafeMetaStoreServer interface {
	// contains filtered or unexported methods
}

UnsafeMetaStoreServer may be embedded to opt out of forward compatibility for this service. Use of this interface is not recommended, as added methods to MetaStoreServer will result in compilation errors.

type UnsafeRaftSurfstoreServer

type UnsafeRaftSurfstoreServer interface {
	// contains filtered or unexported methods
}

UnsafeRaftSurfstoreServer may be embedded to opt out of forward compatibility for this service. Use of this interface is not recommended, as added methods to RaftSurfstoreServer will result in compilation errors.

type UpdateOperation

type UpdateOperation struct {
	Term         int64         `protobuf:"varint,1,opt,name=term,proto3" json:"term,omitempty"`
	FileMetaData *FileMetaData `protobuf:"bytes,2,opt,name=fileMetaData,proto3" json:"fileMetaData,omitempty"`
	// contains filtered or unexported fields
}

func (*UpdateOperation) Descriptor deprecated

func (*UpdateOperation) Descriptor() ([]byte, []int)

Deprecated: Use UpdateOperation.ProtoReflect.Descriptor instead.

func (*UpdateOperation) GetFileMetaData

func (x *UpdateOperation) GetFileMetaData() *FileMetaData

func (*UpdateOperation) GetTerm

func (x *UpdateOperation) GetTerm() int64

func (*UpdateOperation) ProtoMessage

func (*UpdateOperation) ProtoMessage()

func (*UpdateOperation) ProtoReflect

func (x *UpdateOperation) ProtoReflect() protoreflect.Message

func (*UpdateOperation) Reset

func (x *UpdateOperation) Reset()

func (*UpdateOperation) String

func (x *UpdateOperation) String() string

type Version

type Version struct {
	Version int32 `protobuf:"varint,1,opt,name=version,proto3" json:"version,omitempty"`
	// contains filtered or unexported fields
}

func (*Version) Descriptor deprecated

func (*Version) Descriptor() ([]byte, []int)

Deprecated: Use Version.ProtoReflect.Descriptor instead.

func (*Version) GetVersion

func (x *Version) GetVersion() int32

func (*Version) ProtoMessage

func (*Version) ProtoMessage()

func (*Version) ProtoReflect

func (x *Version) ProtoReflect() protoreflect.Message

func (*Version) Reset

func (x *Version) Reset()

func (*Version) String

func (x *Version) String() string

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL