fts5

package main

import (
	"errors"
	"fmt"
	"log"
	"unicode"
	"unicode/utf8"

	"github.com/ncruces/go-sqlite3"
	"github.com/ncruces/go-sqlite3/driver"
	"github.com/ncruces/go-sqlite3/ext/fts5"
	"golang.org/x/text/cases"

	_ "github.com/ncruces/go-sqlite3/vfs/memdb"
)

func main() {
	db, err := driver.Open("file:/test.db?vfs=memdb", func(c *sqlite3.Conn) error {
		return fts5.RegisterCustom(c, func(a *fts5.API) error {
			return errors.Join(
				a.CreateTokenizer("utf8", func(arg []string) (fts5.Tokenizer, error) {
					return utf8Tokenizer{}, nil
				}),
				a.CreateFunction("hit_count", func(fts fts5.Context, ctx sqlite3.Context, arg ...sqlite3.Value) {
					if count, err := fts.InstCount(); err != nil {
						ctx.ResultError(err)
					} else {
						ctx.ResultInt(count)
					}
				}))
		})
	})
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	_, err = db.Exec(`
		CREATE VIRTUAL TABLE docs USING fts5(title, body, tokenize=utf8);
		INSERT INTO docs(title, body) VALUES 
			('Go Programming', 'An intensive guide to Go routines.'),
			('SQLite Tutorial', 'Learn how to use virtual tables efficiently.');
	`)
	if err != nil {
		log.Fatal(err)
	}

	var title string
	var hits int
	err = db.QueryRow("SELECT title, hit_count(docs) FROM docs WHERE docs MATCH 'go AND routines'").Scan(&title, &hits)
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("%s: %d hits\n", title, hits)
}

type utf8Tokenizer struct{}

func (utf8Tokenizer) Tokenize(flags fts5.TokenizeFlag, text, locale string, token fts5.TokenCallback) error {
	folder := cases.Fold()
	isToken := func(r rune) bool {
		return unicode.IsLetter(r) || unicode.IsNumber(r) || unicode.Is(unicode.Co, r)
	}

	var start int
	for start < len(text) {
		for start < len(text) {
			r, sz := utf8.DecodeRuneInString(text[start:])
			if isToken(r) {
				break
			}
			start += sz
		}

		end := start
		for end < len(text) {
			r, sz := utf8.DecodeRuneInString(text[end:])
			if !isToken(r) {
				break
			}
			end += sz
		}

		if start < end {
			err := token(0, folder.String(text[start:end]), start, end)
			if err != nil {
				return err
			}
			start = end
		}
	}
	return nil
}

Output:
Go Programming: 3 hits