quickwit

package module
v0.0.0-...-d3efc0e Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 22, 2025 License: Apache-2.0 Imports: 12 Imported by: 0

README

quickwit-go

Tests Go Report Card GoDoc

A Go client library for Quickwit, a cloud-native search engine.

Features

  • ✅ Full Quickwit API support
  • ✅ Index management (create, list, get, delete, clear, describe)
  • ✅ Source management (create, delete)
  • ✅ Search operations
  • ✅ Split operations
  • ✅ Cluster health checks
  • ✅ Elasticsearch-compatible endpoint
  • ✅ Comprehensive test suite with Testcontainers

Installation

go get github.com/CleverCloud/quickwit-go

Quick Start

package main

import (
    "context"
    "log"

    quickwit "github.com/CleverCloud/quickwit-go"
)

func main() {
    // Create a new client
    client := quickwit.New(
        quickwit.WithEndpoint("http://localhost:7280"),
    )

    ctx := context.Background()

    // Create an index
    indexConfig := quickwit.IndexConfig{
        Version: "0.9",
        ID:      "my-index",
        DocMapping: quickwit.DocMapping{
            Mode: "dynamic",
            FieldMappings: []quickwit.FieldMapping{
                {
                    Name:    "message",
                    Type:    "text",
                    Indexed: true,
                    Stored:  true,
                },
            },
        },
        IndexingSettings: quickwit.Settings{
            Resources: quickwit.Resources{
                HeapSize: "100MB",
            },
        },
        SearchSettings: quickwit.SearchSettings{
            DefaultSearchFields: []string{"message"},
        },
    }

    index, err := client.CreateIndex(ctx, indexConfig)
    if err != nil {
        log.Fatal(err)
    }

    log.Printf("Created index: %s", index.Config.ID)

    // Search
    results, err := client.Search(ctx, "my-index", "*")
    if err != nil {
        log.Fatal(err)
    }

    log.Printf("Found %d results", results.NumHits)
}

Client Options

// With custom endpoint
client := quickwit.New(
    quickwit.WithEndpoint("http://quickwit.example.com:7280"),
)

// With authentication
client := quickwit.New(
    quickwit.WithEndpoint("http://quickwit.example.com:7280"),
    quickwit.WithBearerToken("your-token"),
)

// With basic auth
client := quickwit.New(
    quickwit.WithEndpoint("http://quickwit.example.com:7280"),
    quickwit.WithBasicAuth("username", "password"),
)

// With custom HTTP client
httpClient := &http.Client{Timeout: 30 * time.Second}
client := quickwit.New(
    quickwit.WithHttpClient(httpClient),
)

// With logger
logger := logrus.New()
client := quickwit.New(
    quickwit.WithLogger(logger),
)

API Coverage

Cluster Operations
  • GetCluster() - Get cluster information
  • GetElastic() - Get Elasticsearch-compatible endpoint info
Index Operations
  • CreateIndex(ctx, config) - Create a new index
  • ListIndexes(ctx) - List all indexes
  • GetIndex(ctx, indexID) - Get a specific index
  • DeleteIndex(ctx, indexID) - Delete an index
  • ClearIndex(ctx, indexID) - Clear all data from an index
  • DescribeIndex(ctx, indexID) - Get index statistics
  • ListSplits(ctx, indexID) - List index splits
Source Operations
  • CreateSource(ctx, indexID, config) - Create a data source
  • DeleteSource(ctx, indexID, sourceID) - Delete a source
Search Operations
  • Search(ctx, indexID, query) - Execute a search query

Testing

The library includes comprehensive integration tests using Testcontainers.

# Run tests
go test -v ./...

# Run tests with coverage
go test -v -race -coverprofile=coverage.out ./...

# Run tests with container logs
CONTAINER_LOG=true go test -v ./...

See TESTING.md for more details.

Development

Requirements
  • Go 1.24+
  • Docker (for integration tests)
Running Tests Locally
# Install dependencies
go mod download

# Run all tests
go test -v ./...

# Run linter
golangci-lint run

Contributing

Contributions are welcome! Please feel free to submit a Pull Request.

Documentation

Index

Constants

View Source 
const (
	QuickwitImage    = "quickwit/quickwit:edge"
	QuickwitHttpPort = "7280/tcp"
	QuickwitGrpcPort = "7281/tcp"
)
View Source 
const DefaultEndpoint = "http://localhost:7280"

Variables

View Source 
var ContainerLogs = os.Getenv("CONTAINER_LOG") == "true"

Functions

func GetList 

func GetList[T any](log logrus.FieldLogger, req *http.Request) ([]T, error)

func MustMarshall 

func MustMarshall(i any) *bytes.Buffer

func Request 

func Request[T any](log logrus.FieldLogger, req *http.Request) (*T, error)

func RequestNoContent 

func RequestNoContent(log logrus.FieldLogger, req *http.Request) error

func WithBasicAuth 

func WithBasicAuth(user, password string) func(*client)

func WithBearerToken 

func WithBearerToken(token string) func(*client)

func WithBucket 

func WithBucket(bucketName string) func(cfg *IndexConfig)

func WithEndpoint 

func WithEndpoint(endpoint string) func(*client)

Set endpoint must have `http://localhost:7280` pattern

func WithHttpClient 

func WithHttpClient(httpClient *http.Client) func(*client)

func WithLogger 

func WithLogger(logger logrus.FieldLogger) func(*client)

func WithRetention 

func WithRetention(d time.Duration) func(cfg *IndexConfig)

Types

type Client 

type Client interface {
	Search(ctx context.Context, indexID, query string) (*SearchResponse, error)
	// StreamSearchIndex(ctx context.Context, indexID string) error
	ListIndexes(ctx context.Context) ([]Index, error)
	GetIndex(ctx context.Context, indexID string) (*Index, error)
	CreateIndex(ctx context.Context, idx IndexConfig) (*Index, error)
	DeleteIndex(ctx context.Context, indexID string) error
	ClearIndex(ctx context.Context, indexID string) error
	DescribeIndex(ctx context.Context, indexID string) (*Describe, error)
	ListSplits(ctx context.Context, indexID string) (*SplitsRes, error)

	CreateSource(ctx context.Context, idx string, src SourceConfig) (*SourceConfig, error)
	DeleteSource(ctx context.Context, indexID, sourceID string) error

	GetElastic(ctx context.Context) (*Cluster, error)
	GetCluster(ctx context.Context) (*Cluster, error)
}

func New 

func New(opts ...clientOption) Client

type Cluster 

type Cluster struct {
	ClusterID  string `json:"cluster_id"`
	SelfNodeID string `json:"self_node_id"`
	ReadyNodes []struct {
		NodeID              string `json:"node_id"`
		GenerationID        int64  `json:"generation_id"`
		GossipAdvertiseAddr string `json:"gossip_advertise_addr"`
	} `json:"ready_nodes"`
	LiveNodes             []any `json:"live_nodes"`
	DeadNodes             []any `json:"dead_nodes"`
	ChitchatStateSnapshot struct {
		NodeStateSnapshots []struct {
			ChitchatID struct {
				NodeID              string `json:"node_id"`
				GenerationID        int64  `json:"generation_id"`
				GossipAdvertiseAddr string `json:"gossip_advertise_addr"`
			} `json:"chitchat_id"`
			NodeState struct {
				ChitchatID struct {
					NodeID              string `json:"node_id"`
					GenerationID        int64  `json:"generation_id"`
					GossipAdvertiseAddr string `json:"gossip_advertise_addr"`
				} `json:"chitchat_id"`
				Heartbeat int `json:"heartbeat"`
				KeyValues struct {
					EnabledServices struct {
						Value   string `json:"value"`
						Version int    `json:"version"`
						Status  string `json:"status"`
					} `json:"enabled_services"`
					GrpcAdvertiseAddr struct {
						Value   string `json:"value"`
						Version int    `json:"version"`
						Status  string `json:"status"`
					} `json:"grpc_advertise_addr"`
					IndexerTask01JKTRV3MFFPE8C0AVFJAW0JVK struct {
						Value   string `json:"value"`
						Version int    `json:"version"`
						Status  string `json:"status"`
					} `json:"indexer.task:01JKTRV3MFFPE8C0AVFJAW0JVK"`
					IndexerTask01JKTRV3MFNMWYHD016JT86KEF struct {
						Value   string `json:"value"`
						Version int    `json:"version"`
						Status  string `json:"status"`
					} `json:"indexer.task:01JKTRV3MFNMWYHD016JT86KEF"`
					IndexerTask01JKTRV3MFSYM43GAZE2C40ZC5 struct {
						Value   string `json:"value"`
						Version int    `json:"version"`
						Status  string `json:"status"`
					} `json:"indexer.task:01JKTRV3MFSYM43GAZE2C40ZC5"`
					IndexingCPUCapacity struct {
						Value   string `json:"value"`
						Version int    `json:"version"`
						Status  string `json:"status"`
					} `json:"indexing_cpu_capacity"`
					Readiness struct {
						Value   string `json:"value"`
						Version int    `json:"version"`
						Status  string `json:"status"`
					} `json:"readiness"`
				} `json:"key_values"`
				MaxVersion    int `json:"max_version"`
				LastGcVersion int `json:"last_gc_version"`
			} `json:"node_state"`
		} `json:"node_state_snapshots"`
		SeedAddrs []string `json:"seed_addrs"`
	} `json:"chitchat_state_snapshot"`
}

type Describe 

type Describe struct {
	IndexID                       string `json:"index_id" yaml:"index_id"`
	IndexURI                      string `json:"index_uri" yaml:"index_uri"`
	NbPublishedSplits             int    `json:"num_published_splits" yaml:"num_published_splits"`
	SizePublishedSplits           int    `json:"size_published_splits" yaml:"size_published_splits"`
	NbPublishedDocs               int    `json:"num_published_docs" yaml:"num_published_docs"`
	SizePublishedDocsUncompressed int    `json:"size_published_docs_uncompressed" yaml:"size_published_docs_uncompressed"`
	TimestampFieldName            string `json:"timestamp_field_name" yaml:"timestamp_field_name"`
	MinTimestamp                  *int64 `json:"min_timestamp" yaml:"min_timestamp"`
	MaxTimestamp                  *int64 `json:"max_timestamp" yaml:"max_timestamp"`
}

Describe index res body

type DocMapping 

type DocMapping struct {
	FieldMappings      []FieldMapping `json:"field_mappings" yaml:"field_mappings"`
	TagFields          []any          `json:"tag_fields,omitempty" yaml:"tag_fields,omitempty"`
	StoreSource        bool           `json:"store_source,omitempty" yaml:"store_source,omitempty"`
	IndexFieldPresence bool           `json:"index_field_presence,omitempty" yaml:"index_field_presence,omitempty"`
	TimestampField     string         `json:"timestamp_field,omitempty" yaml:"timestamp_field,omitempty"`
	Mode               string         `json:"mode" yaml:"mode"`
	MaxNumPartitions   int            `json:"max_num_partitions,omitempty" yaml:"max_num_partitions,omitempty"`
	Tokenizers         []any          `json:"tokenizers,omitempty" yaml:"tokenizers,omitempty"`
}

type ErrorMsg 

type ErrorMsg struct {
	Message string `json:"message"`
	Error   string `json:"error"`
}

type FieldMapping 

type FieldMapping struct {
	Name          string   `json:"name" yaml:"name"`
	Type          string   `json:"type" yaml:"type"`
	Fast          any      `json:"fast,omitempty" yaml:"fast,omitempty"`
	FastPrecision string   `json:"fast_precision,omitempty" yaml:"fast_precision,omitempty"`
	Indexed       bool     `json:"indexed" yaml:"indexed"`
	InputFormat   string   `json:"input_format,omitempty" yaml:"input_format,omitempty"`   // TODO: accept arrays ?
	InputFormats  []string `json:"input_formats,omitempty" yaml:"input_formats,omitempty"` // TODO: accept arrays ?
	OutputFormat  string   `json:"output_format,omitempty" yaml:"output_format,omitempty"`
	Stored        bool     `json:"stored" yaml:"stored"`
	Fieldnorms    bool     `json:"fieldnorms,omitempty" yaml:"fieldnorms,omitempty"`
	Record        string   `json:"record,omitempty" yaml:"record,omitempty"`
	Tokenizer     string   `json:"tokenizer,omitempty" yaml:"tokenizer,omitempty"`
	Coerce        bool     `json:"coerce,omitempty" yaml:"coerce,omitempty"`
	ExpandDots    bool     `json:"expand_dots,omitempty" yaml:"expand_dots,omitempty"`
}

type FooterOffsets 

type FooterOffsets struct {
	Start int `json:"start"`
	End   int `json:"end"`
}

type Index 

type Index struct {
	Version         string          `json:"version" yaml:"version"`
	UID             string          `json:"index_uid" yaml:"index_uid"`
	Config          IndexConfig     `json:"index_config" yaml:"index_config"`
	Checkpoint      IndexCheckpoint `json:"checkpoint" yaml:"checkpoint"`
	CreateTimestamp UnixTime        `json:"create_timestamp" yaml:"create_timestamp"`
	Sources         []Source        `json:"sources" yaml:"sources"`
}

type IndexCheckpoint 

type IndexCheckpoint map[string]any

type IndexConfig 

type IndexConfig struct {
	Version          string          `json:"version" yaml:"version"`
	ID               string          `json:"index_id" yaml:"index_id"`
	URI              string          `json:"index_uri,omitempty" yaml:"index_uri,omitempty"`
	DocMapping       DocMapping      `json:"doc_mapping" yaml:"doc_mapping"`
	IndexingSettings Settings        `json:"indexing_settings" yaml:"indexing_settings"`
	SearchSettings   SearchSettings  `json:"search_settings" yaml:"search_settings"`
	Retention        *IndexRetention `json:"retention,omitempty" yaml:"retention,omitempty"`
}

type IndexRetention 

type IndexRetention struct {
	// Duration after which splits are dropped, expressed in a human-readable way (1 day, 2 hours, a week, ...).
	Period string `json:"period,omitempty" yaml:"period,omitempty"`
	//Frequency at which the retention policy is evaluated and applied
	// expressed as a cron expression (0 0 * * * *) or human-readable form (hourly, daily, weekly, monthly, yearly).
	Schedule string `json:"schedule,omitempty" yaml:"retention,omitempty"`
}

type Maturity 

type Maturity struct {
	Type                   string `json:"type"`
	MaturationPeriodMillis int64  `json:"maturation_period_millis"`
}

type MergePolicy 

type MergePolicy struct {
	Type             string `json:"type" yaml:"type"`
	MinLevelNumDocs  int    `json:"min_level_num_docs,omitempty" yaml:"min_level_num_docs,omitempty"`
	MergeFactor      int    `json:"merge_factor,omitempty" yaml:"merge_factor,omitempty"`
	MaxMergeFactor   int    `json:"max_merge_factor,omitempty" yaml:"max_merge_factor,omitempty"`
	MaturationPeriod string `json:"maturation_period,omitempty" yaml:"maturation_period,omitempty"`
}

type QuickwitContainer 

type QuickwitContainer struct {
	testcontainers.Container
	Endpoint string
}

QuickwitContainer represents a Quickwit testcontainer instance

func SetupQuickwitContainer 

func SetupQuickwitContainer(ctx context.Context) (*QuickwitContainer, error)

SetupQuickwitContainer starts a Quickwit container for testing

type Resources 

type Resources struct {
	HeapSize string `json:"heap_size,omitempty" yaml:"heap_size,omitempty"`
}

type SearchResponse 

type SearchResponse struct {
	Hits              any `json:"hits"`
	NumHits           int `json:"num_hits"`
	ElapsedTimeMicros int `json:"elapsed_time_micros"`
}

type SearchSettings 

type SearchSettings struct {
	DefaultSearchFields []string `json:"default_search_fields,omitempty" yaml:"default_search_fields,omitempty"`
}

type Settings 

type Settings struct {
	CommitTimeoutSecs        int          `json:"commit_timeout_secs,omitempty" yaml:"commit_timeout_secs,omitempty"`
	DocstoreCompressionLevel int          `json:"docstore_compression_level,omitempty" yaml:"docstore_compression_level,omitempty"`
	DocstoreBlocksize        int          `json:"docstore_blocksize,omitempty" yaml:"docstore_blocksize,omitempty"`
	SplitNumDocsTarget       int          `json:"split_num_docs_target,omitempty" yaml:"split_num_docs_target,omitempty"`
	MergePolicy              *MergePolicy `json:"merge_policy,omitempty" yaml:"merge_policy,omitempty"`
	Resources                Resources    `json:"resources" yaml:"resources"`
}

type Source 

type Source struct {
	ID           string `json:"source_id" yaml:"source_id"`
	Version      string `json:"version" yaml:"version"`
	NumPipelines int    `json:"num_pipelines" yaml:"num_pipelines"`
	Enabled      bool   `json:"enabled" yaml:"enabled"`
	SourceType   string `json:"source_type" yaml:"source_type"`
	InputFormat  string `json:"input_format" yaml:"input_format"`
}

type SourceConfig 

type SourceConfig struct {
	Version       string         `json:"version"`
	ID            string         `json:"source_id"`
	Type          string         `json:"source_type"`
	PipelineCount int            `json:"num_pipelines,omitempty"`
	Params        map[string]any `json:"params"`
}

func NewPulsarSourceConfig 

func NewPulsarSourceConfig(sourceID, endpoint, token, topic string) SourceConfig

type Split 

type Split struct {
	SplitState                  string        `json:"split_state"`
	UpdateTimestamp             int64         `json:"update_timestamp"`
	PublishTimestamp            int64         `json:"publish_timestamp"`
	Version                     string        `json:"version"`
	SplitID                     string        `json:"split_id"`
	IndexUID                    string        `json:"index_uid"`
	PartitionID                 int           `json:"partition_id"`
	SourceID                    string        `json:"source_id"`
	NodeID                      string        `json:"node_id"`
	NumDocs                     int           `json:"num_docs"`
	UncompressedDocsSizeInBytes int           `json:"uncompressed_docs_size_in_bytes"`
	TimeRange                   TimeRange     `json:"time_range"`
	CreateTimestamp             int64         `json:"create_timestamp"`
	Maturity                    Maturity      `json:"maturity"`
	Tags                        []string      `json:"tags"`
	FooterOffsets               FooterOffsets `json:"footer_offsets"`
	DeleteOpstamp               int           `json:"delete_opstamp"`
	NumMergeOps                 int           `json:"num_merge_ops"`
	DocMappingUID               string        `json:"doc_mapping_uid"`
}

type SplitsRes 

type SplitsRes struct {
	Offset int     `json:"offset" yaml:"offset"`
	Size   int     `json:"size" yaml:"size"`
	Splits []Split `json:"splits" yaml:"splits"`
}

type TimeRange 

type TimeRange struct {
	Start int64 `json:"start"`
	End   int64 `json:"end"`
}

type UnixTime 

type UnixTime struct {
	time.Time
}

UnixTime is our magic type

func (*UnixTime) UnmarshalJSON 

func (u *UnixTime) UnmarshalJSON(b []byte) error

UnmarshalJSON is the method that satisfies the Unmarshaller interface

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.