vfsrw

package
v1.0.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 17, 2026 License: Apache-2.0 Imports: 31 Imported by: 0

README

vfsrw

vfsrw (Virtual File System Read-Write) is a virtual file system layer that provides a unified interface for various backend file systems using a consistent naming scheme.

Key Features

  • vfs:/ Prefix: All paths within the virtual filesystem are prefixed with vfs:/, followed by the name of the specific backend (e.g., vfs:/my-storage/path/to/file).
  • Unified Interface: Implements writefs.FullFS, providing consistent access to read and write operations across diverse storage systems.
  • Configurable Backends: Supports multiple backends like OS file systems, S3, SFTP, RemoteFS, and more.
  • Dynamic Configuration: Backends can be configured via a map of configurations.

Path Scheme

Paths are structured as follows: vfs:/<backend-name>/<file-path>

For example, if you have a backend named data and you want to access images/photo.jpg, the full path would be: vfs:/data/images/photo.jpg

Configuration

vfsrw is typically initialized using a Config object, which is a map where keys are backend names and values are VFS configuration objects.

type VFS struct {
	Name        string       `toml:"name"`
	Type        string       `toml:"type"`
	ReadOnly    bool         `toml:"readonly"`
	S3          *S3          `toml:"s3,omitempty"`
	OS          *OS          `toml:"os,omitempty"`
	SFTP        *SFTP        `toml:"sftp,omitempty"`
	Remote      *Remote      `toml:"remote,omitempty"`
	MiniKVStore *MiniKVStore `toml:"minikvstore,omitempty"`
	Web         *Web         `toml:"web,omitempty"`
}

Backend Types

Supported backend types include:

  • os: Local operating system filesystem.
  • s3: Amazon S3 or compatible storage (e.g., MinIO).
  • sftp: Secure File Transfer Protocol.
  • remote: Custom remote filesystem protocol.
  • minikvstore: MiniKVStore-based filesystem.
  • web: Web/HTTP-based read-only (mostly) filesystem.

Each backend may also support zipAsFolder capability if configured.

Backend Configuration Details

Each backend type has its own configuration structure.

os - Local OS Filesystem

Used for local file system access.

  • BaseDir: The root directory of the backend.
  • ZipAsFolderCache: Size of the cache for zipAsFolder capability.
[local]
name = "local"
type = "os"

[local.os]
basedir = "/tmp/data"
zipasfoldercache = 10
s3 - Amazon S3 or Compatible

Used for S3-compatible storage like AWS S3 or MinIO.

  • AccessKeyID: Credentials.
  • SecretAccessKey: Credentials.
  • Endpoint: S3 endpoint URL.
  • Region: S3 region.
  • UseSSL: Enable/disable SSL.
  • Debug: Enable debug logging.
  • CAPEM: Custom CA certificate in PEM format.
  • BaseUrl: Base URL for the S3 service.
  • ZipAsFolderCache: Size of the cache for zipAsFolder capability.
  • DNSNetwork: Network for DNS resolution (e.g., "tcp").
  • DNSAddress: Address for DNS resolution.
[s3-storage]
name = "s3-storage"
type = "s3"

[s3-storage.s3]
accesskeyid = "AKIA..."
secretaccesskey = "..."
endpoint = "s3.amazonaws.com"
region = "us-east-1"
usessl = true
sftp - Secure File Transfer Protocol

Used for remote file access via SFTP.

  • Address: SFTP server address (host:port).
  • User: SFTP username.
  • Password: SFTP password.
  • PrivateKey: Paths to private key files.
  • KnownHosts: List of known hosts.
  • BaseDir: Root directory on the SFTP server.
  • Sessions: Number of concurrent sessions.
  • ZipAsFolderCache: Size of the cache for zipAsFolder capability.
[remote-sftp]
name = "remote-sftp"
type = "sftp"

[remote-sftp.sftp]
address = "sftp.example.com:22"
user = "myuser"
password = "..."
basedir = "/home/myuser"
remote - RemoteFS Protocol

A custom gRPC-based remote filesystem protocol.

  • Address: RemoteFS server address.
  • ClientTLS: TLS configuration for the client.
  • BaseDir: Root directory on the remote server.
  • JWTKey: JWT key for authentication.
  • InsecureSkipVerify: Skip TLS certificate verification.
  • ca: Custom CA certificates.
[remotefs]
name = "remotefs"
type = "remote"

[remotefs.remote]
address = "remotefs.example.com:50051"
jwtkey = "..."
minikvstore - MiniKVStore-based Filesystem

A filesystem backend using MiniKVStore for storage.

  • ResolverAddr: MiniResolver address.
  • ResolverTimeout: Timeout for resolution.
  • ResolverNotFoundTimeout: Timeout for not found results.
  • Domain: Domain in MiniKVStore.
  • ClientTLS: TLS configuration.
  • Dir: Base directory within the store.
[kv-store]
name = "kv-store"
type = "minikvstore"

[kv-store.minikvstore]
resolveraddr = "resolver.example.com"
domain = "mydomain"
web - Web/HTTP Filesystem

A read-only filesystem backend over HTTP.

  • baseuri: Base URL of the web server.
  • header: Custom HTTP headers to include in requests.
  • tls_insecure_skip_verify: Skip TLS certificate verification.
[web-site]
name = "web-site"
type = "web"

[web-site.web]
baseuri = "https://example.com/files/"

Usage

import (
	"github.com/je4/filesystem/v3/pkg/vfsrw"
	"github.com/je4/utils/v2/pkg/zLogger"
)

// Example configuration
cfg := vfsrw.Config{
	"local": &vfsrw.VFS{
		Type: "os",
		OS: &vfsrw.OS{BaseDir: "/tmp/data"},
	},
}

// Initialize VFS
vfs, err := vfsrw.NewFS(cfg, logger)
if err != nil {
	// handle error
}

// Open a file
f, err := vfs.Open("vfs:/local/myfile.txt")

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func AddLocal 

func AddLocal(fSys fs.FS) error

func MatchPath 

func MatchPath(vfsPath string) (name string, path string, err error)

func NewFSWithMiniResolver 

func NewFSWithMiniResolver(config Config, miniResolverClient *resolver.MiniResolver, logger zLogger.ZLogger) (*vFSRW, error)

Types

type AESConfig 

type AESConfig struct {
	Enable       bool             `toml:"enable"`
	KeepassFile  config.EnvString `toml:"keepassfile"`
	KeepassEntry config.EnvString `toml:"keepassentry"`
	KeepassKey   config.EnvString `toml:"keepasskey"`
	IV           config.EnvString `toml:"iv"`
}

type Afero 

type Afero struct {
	BaseDir string `toml:"basedir"`
}

type Config 

type Config map[string]*VFS

type MiniKVStore 

type MiniKVStore struct {
	ResolverAddr            string          `toml:"resolveraddr"`
	ResolverTimeout         config.Duration `toml:"resolvertimeout"`
	ResolverNotFoundTimeout config.Duration `toml:"resolvernotfoundtimeout"`
	Domain                  string          `toml:"domain"`
	ClientTLS               *loader.Config  `toml:"clienttls"`
	Dir                     string          `toml:"dir"`
}

type OS 

type OS struct {
	BaseDir string `toml:"basedir"`
}

type Remote 

type Remote struct {
	CAs                []certconfig.Certificate `toml:"ca"`
	Address            string                   `toml:"address"`
	ClientTLS          *loader.Config           `toml:"clienttls"`
	BaseDir            string                   `toml:"basedir"`
	JWTKey             config.EnvString         `toml:"jwtkey"`
	InsecureSkipVerify bool                     `toml:"insecureskipverify"`
}

type S3 

type S3 struct {
	AccessKeyID     config.EnvString `toml:"accesskeyid"`
	SecretAccessKey config.EnvString `toml:"secretaccesskey"`
	Endpoint        config.EnvString `toml:"endpoint"`
	Region          config.EnvString `toml:"region"`
	UseSSL          bool             `toml:"usessl"`
	Debug           bool             `toml:"debug"`
	CAPEM           string           `toml:"capem"`
	BaseUrl         string           `toml:"baseurl"`
	DNSNetwork      string           `toml:"dnsnetwork"`
	DNSAddress      string           `toml:"dnsaddress"`
}

type SFTP 

type SFTP struct {
	Address    config.EnvString `toml:"address"`
	KnownHosts []string         `toml:"knownhosts"`
	BaseDir    string           `toml:"basedir"`
	Sessions   uint             `toml:"sessions"`
	User       config.EnvString `toml:"user"`
	Password   config.EnvString `toml:"password"`
	PrivateKey []string         `toml:"privatekey"`
}

type VFS 

type VFS struct {
	Name        string       `toml:"name"`
	Type        string       `toml:"type"`
	ReadOnly    bool         `toml:"readonly"`
	ZipAsFolder *ZipAsFolder `toml:"zipasfolder"`
	S3          *S3          `toml:"s3,omitempty"`
	OS          *OS          `toml:"os,omitempty"`
	SFTP        *SFTP        `toml:"sftp,omitempty"`
	Remote      *Remote      `toml:"remote,omitempty"`
	MiniKVStore *MiniKVStore `toml:"minikvstore,omitempty"`
	Web         *Web         `toml:"web,omitempty"`
	Afero       *Afero       `toml:"afero,omitempty"`
}

type VFSRW 

type VFSRW interface {
	fmt.Stringer
	writefs.CopyFS
	writefs.CreateFS
	writefs.MkDirFS
	writefs.RenameFS
	writefs.RemoveFS
	writefs.CloseFS
	writefs.EqualFS
	writefs.IsWriteableFS
	writefs.SubFS
	writefs.SubCreateFS
	writefs.RealPathFS
	writefs.IsEmptyFS
	fs.FS
	fs.ReadDirFS
	fs.ReadFileFS
	fs.StatFS
	AddFS(name string, vfsConfig *VFS, fsys fs.FS)
}

func NewFS 

func NewFS(config Config, logger zLogger.ZLogger) (VFSRW, error)

type Web 

type Web struct {
	BaseURI               string              `toml:"baseuri"`
	Header                map[string][]string `toml:"header"`
	TLSInsecureSkipVerify bool                `toml:"tls_insecure_skip_verify"`
}

type ZipAsFolder 

type ZipAsFolder struct {
	Enabled   bool                       `toml:"enabled"`
	Digests   []checksum.DigestAlgorithm `toml:"digests"`
	CacheSize uint                       `toml:"cachesize"`
	Timeout   config.Duration            `toml:"timeout"`
	Compress  bool                       `toml:"compress"`
	ReadOnly  bool                       `toml:"readonly"`
	AES       *AESConfig                 `toml:"aes"`
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.