repository

package
v0.1.3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 27, 2026 License: MIT Imports: 18 Imported by: 0

Documentation

Overview

Package repository provides repository file indexing and search functionality. It uses SQLite with FTS5 for fast file search across the codebase.

Index

Constants

View Source 
const (
	// MaxFileSize is the maximum file size to index (100MB).
	MaxFileSize = 100 * 1024 * 1024
	// MaxTotalIndexSize is the maximum total size of indexed content (1GB).
	MaxTotalIndexSize = 1 * 1024 * 1024 * 1024
	// MaxFilesPerScan is the maximum number of files to scan.
	MaxFilesPerScan = 1000000
	// ScanTimeout is the maximum time for a full scan.
	ScanTimeout = 5 * time.Minute
	// DefaultSearchLimit is the default number of search results.
	DefaultSearchLimit = 50
	// MaxSearchLimit is the maximum number of search results.
	MaxSearchLimit = 500
	// DefaultListLimit is the default number of files in a list.
	DefaultListLimit = 100
	// MaxListLimit is the maximum number of files in a list.
	MaxListLimit = 1000
	// MaxTreeDepth is the maximum depth for tree queries.
	MaxTreeDepth = 10
)

Security constants

Variables

View Source 
var (
	ErrPathTraversal      = errors.New("path traversal detected")
	ErrSymlinkOutsideRepo = errors.New("symlink points outside repository")
	ErrTooManyFiles       = errors.New("too many files to scan")
	ErrIndexTooLarge      = errors.New("total index size exceeds limit")
	ErrScanTimeout        = errors.New("scan timeout exceeded")
	ErrInvalidPath        = errors.New("invalid path")
)

Scanner errors

View Source 
var BinaryExtensions = []string{
	".exe", ".dll", ".so", ".dylib", ".a", ".o", ".obj",
	".zip", ".tar", ".gz", ".bz2", ".xz", ".7z", ".rar",
	".png", ".jpg", ".jpeg", ".gif", ".bmp", ".ico", ".webp", ".svg",
	".mp3", ".mp4", ".avi", ".mov", ".mkv", ".wav", ".flac",
	".pdf", ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx",
	".wasm", ".pyc", ".pyo", ".class", ".jar", ".war",
	".ttf", ".otf", ".woff", ".woff2", ".eot",
	".db", ".sqlite", ".sqlite3",
}

Binary file extensions to skip content indexing.

View Source 
var SensitivePatterns = []string{
	".env",
	".env.*",
	"*.env",
	"credentials.json",
	"credentials.yaml",
	"credentials.yml",
	"secrets.json",
	"secrets.yaml",
	"secrets.yml",
	"*.key",
	"*.pem",
	"*.p12",
	"*.pfx",
	"*.jks",
	"id_rsa",
	"id_dsa",
	"id_ecdsa",
	"id_ed25519",
	".aws/credentials",
	".netrc",
	".npmrc",
	".pypirc",
}

Sensitive file patterns that should be flagged.

View Source 
var SkipDirectories = config.DefaultSkipDirectories

SkipDirectories references the centralized default list from config. This is kept for backwards compatibility - prefer using config.DefaultSkipDirectories directly. Users can override via config.yaml indexer.skip_directories setting.

Functions

func GetFileID 

func GetFileID(path string) (uint64, error)

GetFileID returns the unique file identifier (inode) for a file. On Unix systems, this is the inode number which persists across renames.

func GetFileIDFromInfo 

func GetFileIDFromInfo(info os.FileInfo) uint64

GetFileIDFromInfo extracts the file ID from existing FileInfo.

Types

type DirectoryInfo 

type DirectoryInfo struct {
	Path           string    `json:"path"`
	Name           string    `json:"name"`
	FileCount      int       `json:"file_count"`
	TotalSizeBytes int64     `json:"total_size_bytes"`
	LastModified   time.Time `json:"last_modified,omitempty"`
}

DirectoryInfo represents metadata about a directory.

type DirectoryTree 

type DirectoryTree struct {
	Path           string          `json:"path"`
	Name           string          `json:"name"`
	Type           string          `json:"type"` // "file" or "directory"
	Children       []DirectoryTree `json:"children,omitempty"`
	SizeBytes      *int64          `json:"size_bytes,omitempty"`
	Extension      *string         `json:"extension,omitempty"`
	FileCount      *int            `json:"file_count,omitempty"`
	TotalSizeBytes *int64          `json:"total_size_bytes,omitempty"`
}

DirectoryTree represents a hierarchical directory structure.

type FileInfo 

type FileInfo struct {
	ID          int64     `json:"-"`
	Path        string    `json:"path"`
	Name        string    `json:"name"`
	Directory   string    `json:"directory"`
	Extension   string    `json:"extension,omitempty"`
	SizeBytes   int64     `json:"size_bytes"`
	ModifiedAt  time.Time `json:"modified_at"`
	IndexedAt   time.Time `json:"indexed_at,omitempty"`
	IsBinary    bool      `json:"is_binary"`
	IsSymlink   bool      `json:"is_symlink"`
	IsSensitive bool      `json:"is_sensitive"`
	GitTracked  bool      `json:"git_tracked"`
	GitIgnored  bool      `json:"git_ignored"`
	ContentHash string    `json:"-"`
	LineCount   int       `json:"line_count,omitempty"`
	MatchScore  float64   `json:"match_score,omitempty"` // For search results
	FileID      uint64    `json:"-"`                     // Inode (Unix) or File ID (Windows) for rename detection
}

FileInfo represents metadata about a file in the repository.

type FileList 

type FileList struct {
	Directory   string          `json:"directory"`
	Files       []FileInfo      `json:"files"`
	Directories []DirectoryInfo `json:"directories"`
	TotalFiles  int             `json:"total_files"`
	TotalDirs   int             `json:"total_directories"`
	Pagination  PaginationInfo  `json:"pagination"`
}

FileList contains a paginated list of files.

type IndexStatus 

type IndexStatus struct {
	Status            string    `json:"status"` // ready, indexing, error
	TotalFiles        int       `json:"total_files"`
	IndexedFiles      int       `json:"indexed_files"`
	TotalSizeBytes    int64     `json:"total_size_bytes"`
	LastFullScan      time.Time `json:"last_full_scan"`
	LastUpdate        time.Time `json:"last_update"`
	DatabaseSizeBytes int64     `json:"database_size_bytes"`
	IsGitRepo         bool      `json:"is_git_repo"`
	ErrorMessage      string    `json:"error_message,omitempty"`
}

IndexStatus represents the current state of the repository index.

type Indexer 

type Indexer interface {
	// Lifecycle
	Start(ctx context.Context) error
	Stop() error
	IsReady() bool

	// Indexing
	FullScan(ctx context.Context) error
	IndexFile(ctx context.Context, path string) error
	RemoveFile(ctx context.Context, path string) error

	// Queries
	Search(ctx context.Context, query SearchQuery) (*SearchResult, error)
	ListFiles(ctx context.Context, opts ListOptions) (*FileList, error)
	GetTree(ctx context.Context, rootPath string, depth int) (*DirectoryTree, error)
	GetStats(ctx context.Context) (*RepositoryStats, error)

	// Status
	GetStatus() IndexStatus
}

Indexer defines the interface for repository indexing operations.

type ListOptions 

type ListOptions struct {
	Directory  string   `json:"directory"`
	Recursive  bool     `json:"recursive"`
	Limit      int      `json:"limit"`
	Offset     int      `json:"offset"`
	SortBy     string   `json:"sort_by"`    // name, size, modified
	SortOrder  string   `json:"sort_order"` // asc, desc
	Extensions []string `json:"extensions,omitempty"`
	MinSize    int64    `json:"min_size,omitempty"`
	MaxSize    int64    `json:"max_size,omitempty"`
}

ListOptions defines parameters for listing files.

type PaginationInfo 

type PaginationInfo struct {
	Limit   int  `json:"limit"`
	Offset  int  `json:"offset"`
	HasMore bool `json:"has_more"`
}

PaginationInfo contains pagination metadata.

type RepositoryStats 

type RepositoryStats struct {
	TotalFiles       int            `json:"total_files"`
	TotalDirectories int            `json:"total_directories"`
	TotalSizeBytes   int64          `json:"total_size_bytes"`
	FilesByExtension map[string]int `json:"files_by_extension"`
	LargestFiles     []FileInfo     `json:"largest_files"`
	GitTrackedFiles  int            `json:"git_tracked_files"`
	GitIgnoredFiles  int            `json:"git_ignored_files"`
	BinaryFiles      int            `json:"binary_files"`
	SensitiveFiles   int            `json:"sensitive_files"`
}

RepositoryStats contains aggregate statistics about the repository.

type SQLiteIndexer 

type SQLiteIndexer struct {
	// contains filtered or unexported fields
}

SQLiteIndexer implements the Indexer interface using SQLite with FTS5.

func NewIndexer 

func NewIndexer(repoPath string, skipDirs []string) (*SQLiteIndexer, error)

NewIndexer creates a new SQLite-based repository indexer. If skipDirs is nil or empty, the default SkipDirectories list is used.

func (*SQLiteIndexer) FullScan 

func (idx *SQLiteIndexer) FullScan(ctx context.Context) error

FullScan performs a complete scan of the repository.

func (*SQLiteIndexer) GetStats 

func (idx *SQLiteIndexer) GetStats(ctx context.Context) (*RepositoryStats, error)

GetStats returns aggregate statistics about the repository. Uses caching with event-driven invalidation for better performance.

func (*SQLiteIndexer) GetStatus 

func (idx *SQLiteIndexer) GetStatus() IndexStatus

GetStatus returns the current index status.

func (*SQLiteIndexer) GetTree 

func (idx *SQLiteIndexer) GetTree(ctx context.Context, rootPath string, depth int) (*DirectoryTree, error)

GetTree returns a hierarchical directory structure.

func (*SQLiteIndexer) IndexFile 

func (idx *SQLiteIndexer) IndexFile(ctx context.Context, relPath string) error

IndexFile indexes or updates a single file. It also detects renames by checking if a file with the same file_id exists at a different path.

func (*SQLiteIndexer) InvalidateStatsCache 

func (idx *SQLiteIndexer) InvalidateStatsCache()

InvalidateStatsCache invalidates the cached repository statistics. This should be called when files are added, modified, or removed.

func (*SQLiteIndexer) IsReady 

func (idx *SQLiteIndexer) IsReady() bool

IsReady returns true if the indexer is ready for queries.

func (*SQLiteIndexer) ListFiles 

func (idx *SQLiteIndexer) ListFiles(ctx context.Context, opts ListOptions) (*FileList, error)

ListFiles returns a paginated list of files in a directory.

func (*SQLiteIndexer) RemoveFile 

func (idx *SQLiteIndexer) RemoveFile(ctx context.Context, relPath string) error

RemoveFile removes a file from the index.

func (*SQLiteIndexer) Search 

func (idx *SQLiteIndexer) Search(ctx context.Context, query SearchQuery) (*SearchResult, error)

Search performs a file search based on the given query.

func (*SQLiteIndexer) Start 

func (idx *SQLiteIndexer) Start(ctx context.Context) error

Start begins the indexer and performs initial scan.

func (*SQLiteIndexer) Stop 

func (idx *SQLiteIndexer) Stop() error

Stop gracefully stops the indexer.

type Scanner 

type Scanner struct {
	// contains filtered or unexported fields
}

Scanner handles file system scanning with security validations.

func NewScanner 

func NewScanner(repoPath string, customSkipDirs []string) (*Scanner, error)

NewScanner creates a new file scanner for the given repository path. If customSkipDirs is nil or empty, the default SkipDirectories list is used.

func (*Scanner) IsGitRepo 

func (s *Scanner) IsGitRepo() bool

IsGitRepo checks if the repository path is a git repository.

func (*Scanner) RepoPath 

func (s *Scanner) RepoPath() string

RepoPath returns the absolute repository path.

func (*Scanner) ScanAll 

func (s *Scanner) ScanAll(ctx context.Context) ([]FileInfo, error)

ScanAll performs a full scan of the repository.

func (*Scanner) ScanFile 

func (s *Scanner) ScanFile(ctx context.Context, relPath string) (*FileInfo, error)

ScanFile scans a single file and returns its metadata.

func (*Scanner) ValidatePath 

func (s *Scanner) ValidatePath(relPath string) error

ValidatePath ensures the path is within the repository and safe.

type SearchMode 

type SearchMode string

SearchMode defines the type of search to perform. 

const (
	// SearchModeFuzzy uses FTS5 for fuzzy matching.
	SearchModeFuzzy SearchMode = "fuzzy"
	// SearchModeExact matches the exact query string.
	SearchModeExact SearchMode = "exact"
	// SearchModePrefix matches files starting with the query.
	SearchModePrefix SearchMode = "prefix"
	// SearchModeExtension filters by file extension.
	SearchModeExtension SearchMode = "extension"
)

type SearchQuery 

type SearchQuery struct {
	Query           string     `json:"query"`
	Mode            SearchMode `json:"mode"`
	Limit           int        `json:"limit"`
	Offset          int        `json:"offset"`
	Extensions      []string   `json:"extensions,omitempty"`
	ExcludeBinaries bool       `json:"exclude_binaries"`
	GitTrackedOnly  bool       `json:"git_tracked_only"`
	MinSize         int64      `json:"min_size,omitempty"`
	MaxSize         int64      `json:"max_size,omitempty"`
}

SearchQuery defines parameters for searching files.

type SearchResult 

type SearchResult struct {
	Query     string     `json:"query"`
	Mode      SearchMode `json:"mode"`
	Results   []FileInfo `json:"results"`
	Total     int        `json:"total"`
	ElapsedMS int64      `json:"elapsed_ms"`
}

SearchResult contains the results of a file search.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.