Add full-text search

- Index notes in a separate, rebuildable SQLite database (index.db) using FTS5 trigram,
  synced from watcher snapshots so external edits are picked up; terms under 3 characters
  fall back to LIKE
- GET /api/v1/search?q= returns matching notes with the first matching line and a snippet;
  path matches rank first
- Sidebar search panel (Ctrl+Shift+F) with highlighted terms and keyboard navigation;
  opening a result jumps to and selects the match
- Restore scroll position on tab switch with scroll snapshots; restoring scrollTop drifted
  by a few lines after line heights were re-estimated
This commit is contained in:
eezd committed 2026-10-16 20:17:24 +08:00
1 parent a11e0ec498
commit 55f0ee174e
13 files changed
+792 -20

No files matched your search

+16 -2
View File
@@ -16,6 +16,7 @@ import (
"rawmark/internal/auth"
"rawmark/internal/events"
"rawmark/internal/search"
"rawmark/internal/vault"
)
@@ -30,12 +31,13 @@ type Server struct {
vault *vault.Vault
auth *auth.Manager
hub *events.Hub
index *search.Index
// OnSetup 在首次初始化完成后调用(用于删除启动码文件)
OnSetup func()
}
func New(v *vault.Vault, a *auth.Manager, hub *events.Hub) *Server {
return &Server{vault: v, auth: a, hub: hub}
func New(v *vault.Vault, a *auth.Manager, hub *events.Hub, index *search.Index) *Server {
return &Server{vault: v, auth: a, hub: hub, index: index}
}
func (s *Server) Handler(static fs.FS) http.Handler {
@@ -59,6 +61,7 @@ func (s *Server) Handler(static fs.FS) http.Handler {
"POST /api/v1/trash/{id}/restore": s.trashRestore,
"DELETE /api/v1/trash/{id}": s.trashPurge,
"GET /api/v1/events": s.events,
"GET /api/v1/search": s.search,
}
for pattern, h := range protected {
mux.Handle(pattern, s.requireAuth(h))
@@ -211,6 +214,17 @@ func (s *Server) tree(w http.ResponseWriter, r *http.Request) {
writeJSON(w, 200, map[string]any{"entries": entries})
}
const searchLimit = 50
func (s *Server) search(w http.ResponseWriter, r *http.Request) {
results, err := s.index.Search(r.URL.Query().Get("q"), searchLimit)
if err != nil {
internalError(w, r, err)
return
}
writeJSON(w, 200, map[string]any{"results": results})
}
func (s *Server) readFile(w http.ResponseWriter, r *http.Request) {
data, rev, err := s.vault.Read(r.PathValue("path"))
if err != nil {
+310
View File
@@ -0,0 +1,310 @@
// Package search 维护笔记的全文索引。索引存放在独立的数据库文件中,内容完全由仓库文件派生,
// 删除后会在下次启动时重建;结构变化时也直接重建而不做迁移。
package search
import (
"context"
"database/sql"
"errors"
"fmt"
"log/slog"
"strings"
"sync"
"unicode"
"unicode/utf8"
_ "modernc.org/sqlite"
"rawmark/internal/vault"
)
// 修改表结构时递增,旧索引会被丢弃重建
const schemaVersion = 1
const schema = `
CREATE TABLE docs (
id INTEGER PRIMARY KEY,
path TEXT NOT NULL UNIQUE,
rev TEXT NOT NULL
);
-- trigram 分词不依赖空格断词,中文可按任意片段检索;不足 3 个字符的词改用 LIKE
CREATE VIRTUAL TABLE docs_fts USING fts5(path, body, tokenize = 'trigram');
`
const (
maxTerms = 8
snippetRunes = 160
snippetBefore = 40
)
type Result struct {
Path string `json:"path"`
// 第一处匹配所在行(从 1 开始)与该行的摘录;只有文件名匹配时 Line 为 0
Line int `json:"line"`
Snippet string `json:"snippet"`
}
type doc struct {
id int64
rev string
}
type Index struct {
db *sql.DB
v *vault.Vault
mu sync.Mutex
pending map[string]string // 待同步的最新快照(路径 -> rev)
wake chan struct{}
known map[string]doc // 已索引的笔记,仅由同步协程访问
}
func Open(path string, v *vault.Vault) (*Index, error) {
db, err := sql.Open("sqlite", "file:"+path+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)")
if err != nil {
return nil, err
}
x := &Index{db: db, v: v, wake: make(chan struct{}, 1), known: map[string]doc{}}
if err := x.init(); err != nil {
db.Close()
return nil, fmt.Errorf("初始化搜索索引失败: %w", err)
}
return x, nil
}
func (x *Index) init() error {
var version int
if err := x.db.QueryRow(`PRAGMA user_version`).Scan(&version); err != nil {
return err
}
if version != schemaVersion {
stmts := []string{`DROP TABLE IF EXISTS docs`, `DROP TABLE IF EXISTS docs_fts`, schema, fmt.Sprintf(`PRAGMA user_version = %d`, schemaVersion)}
for _, s := range stmts {
if _, err := x.db.Exec(s); err != nil {
return err
}
}
}
rows, err := x.db.Query(`SELECT id, path, rev FROM docs`)
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var d doc
var p string
if err := rows.Scan(&d.id, &p, &d.rev); err != nil {
return err
}
x.known[p] = d
}
return rows.Err()
}
func (x *Index) Close() error { return x.db.Close() }
// Update 提交仓库的最新快照(笔记路径 -> rev),立即返回;同步在后台进行,多次提交只处理最新一次
func (x *Index) Update(files map[string]string) {
x.mu.Lock()
x.pending = files
x.mu.Unlock()
select {
case x.wake <- struct{}{}:
default:
}
}
// Run 执行后台同步,直到 ctx 结束
func (x *Index) Run(ctx context.Context) {
for {
select {
case <-ctx.Done():
return
case <-x.wake:
x.mu.Lock()
files := x.pending
x.mu.Unlock()
if err := x.sync(files); err != nil {
slog.Error("更新搜索索引失败", "err", err)
}
}
}
}
// sync 让索引与快照一致:只读取新增或 rev 变化的笔记
func (x *Index) sync(files map[string]string) error {
tx, err := x.db.Begin()
if err != nil {
return err
}
defer tx.Rollback()
next := make(map[string]doc, len(x.known))
for p, d := range x.known {
next[p] = d
}
for p, d := range x.known {
if _, ok := files[p]; !ok {
if err := remove(tx, d.id); err != nil {
return err
}
delete(next, p)
}
}
for p, rev := range files {
old, ok := x.known[p]
if ok && old.rev == rev {
continue
}
content, rev, err := x.v.Read(p)
if errors.Is(err, vault.ErrNotFound) {
continue // 快照之后被删除,下一次快照会处理
}
if err != nil {
return err
}
if ok {
if err := remove(tx, old.id); err != nil {
return err
}
}
res, err := tx.Exec(`INSERT INTO docs (path, rev) VALUES (?, ?)`, p, rev)
if err != nil {
return err
}
id, err := res.LastInsertId()
if err != nil {
return err
}
if _, err := tx.Exec(`INSERT INTO docs_fts (rowid, path, body) VALUES (?, ?, ?)`, id, p, string(content)); err != nil {
return err
}
next[p] = doc{id: id, rev: rev}
}
if err := tx.Commit(); err != nil {
return err
}
x.known = next
return nil
}
func remove(tx *sql.Tx, id int64) error {
if _, err := tx.Exec(`DELETE FROM docs WHERE id = ?`, id); err != nil {
return err
}
_, err := tx.Exec(`DELETE FROM docs_fts WHERE rowid = ?`, id)
return err
}
// Search 查找同时包含所有词(空白分隔,不区分大小写)的笔记,词可出现在路径或正文中。
// 路径(文件名或目录名)命中第一个词的排在前面,其余按相关度排序
func (x *Index) Search(query string, limit int) ([]Result, error) {
terms := strings.Fields(query)
if len(terms) == 0 {
return []Result{}, nil
}
if len(terms) > maxTerms {
terms = terms[:maxTerms]
}
var where, phrases []string
var args []any
for _, t := range terms {
if utf8.RuneCountInString(t) >= 3 {
phrases = append(phrases, `"`+strings.ReplaceAll(t, `"`, `""`)+`"`)
} else {
like := "%" + escapeLike(t) + "%"
where = append(where, `(path LIKE ? ESCAPE '\' OR body LIKE ? ESCAPE '\')`)
args = append(args, like, like)
}
}
order := "path"
if len(phrases) > 0 {
where = append(where, `docs_fts MATCH ?`)
args = append(args, strings.Join(phrases, " AND "))
order = "rank"
}
name := "%" + escapeLike(terms[0]) + "%"
q := `SELECT path, body FROM docs_fts WHERE ` + strings.Join(where, " AND ") +
` ORDER BY path LIKE ? ESCAPE '\' DESC, ` + order + ` LIMIT ?`
args = append(args, name, limit)
rows, err := x.db.Query(q, args...)
if err != nil {
return nil, err
}
defer rows.Close()
results := []Result{}
for rows.Next() {
var r Result
var body string
if err := rows.Scan(&r.Path, &body); err != nil {
return nil, err
}
r.Line, r.Snippet = snippet(body, terms)
results = append(results, r)
}
return results, rows.Err()
}
func escapeLike(s string) string {
return strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(s)
}
// snippet 返回正文中第一处匹配所在的行号与摘录(过长的行截取匹配附近的一段)
func snippet(body string, terms []string) (int, string) {
folded := make([][]rune, len(terms))
for i, t := range terms {
folded[i] = fold([]rune(t))
}
for n, line := range strings.Split(body, "\n") {
runes := []rune(strings.TrimRight(line, "\r"))
lower := fold(runes)
at := -1
for _, t := range folded {
if i := indexRunes(lower, t); i >= 0 && (at < 0 || i < at) {
at = i
}
}
if at < 0 {
continue
}
start := 0
if len(runes) > snippetRunes {
start = max(0, min(at-snippetBefore, len(runes)-snippetRunes))
}
end := min(len(runes), start+snippetRunes)
text := strings.TrimSpace(string(runes[start:end]))
if start > 0 {
text = "…" + text
}
if end < len(runes) {
text += "…"
}
return n + 1, text
}
return 0, ""
}
// fold 逐字符转小写,保持字符数不变,便于在原文中按位置截取
func fold(rs []rune) []rune {
out := make([]rune, len(rs))
for i, r := range rs {
out[i] = unicode.ToLower(r)
}
return out
}
func indexRunes(s, sub []rune) int {
for i := 0; i+len(sub) <= len(s); i++ {
match := true
for j := range sub {
if s[i+j] != sub[j] {
match = false
break
}
}
if match {
return i
}
}
return -1
}
+181
View File
@@ -0,0 +1,181 @@
package search
import (
"os"
"path/filepath"
"strings"
"testing"
"rawmark/internal/vault"
)
type fixture struct {
t *testing.T
v *vault.Vault
idx *Index
db string
}
func newFixture(t *testing.T) *fixture {
t.Helper()
v, err := vault.Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
f := &fixture{t: t, v: v, db: filepath.Join(t.TempDir(), "index.db")}
f.reopen()
return f
}
func (f *fixture) reopen() {
if f.idx != nil {
f.idx.Close()
}
idx, err := Open(f.db, f.v)
if err != nil {
f.t.Fatal(err)
}
f.idx = idx
f.t.Cleanup(func() { idx.Close() })
}
func (f *fixture) write(p, content string) {
abs := filepath.Join(f.v.Root(), p)
if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil {
f.t.Fatal(err)
}
if err := os.WriteFile(abs, []byte(content), 0o644); err != nil {
f.t.Fatal(err)
}
}
// sync 模拟监听器提交快照
func (f *fixture) sync() {
f.t.Helper()
entries, err := f.v.Tree()
if err != nil {
f.t.Fatal(err)
}
files := map[string]string{}
for _, e := range entries {
if !e.Dir {
data, _, err := f.v.Read(e.Path)
if err != nil {
f.t.Fatal(err)
}
files[e.Path] = vault.Rev(data)
}
}
if err := f.idx.sync(files); err != nil {
f.t.Fatal(err)
}
}
func (f *fixture) paths(q string) string {
f.t.Helper()
rs, err := f.idx.Search(q, 50)
if err != nil {
f.t.Fatalf("搜索 %q: %v", q, err)
}
var out []string
for _, r := range rs {
out = append(out, r.Path)
}
return strings.Join(out, ",")
}
func TestSearchTerms(t *testing.T) {
f := newFixture(t)
f.write("a.md", "# 标题\n\n今天学习 CodeMirror 的装饰器\n")
f.write("b.md", "第二篇笔记,讲 SQLite 全文检索\n")
f.write("目录/c.md", "装饰器和检索都提到\n")
f.sync()
cases := map[string]string{
"装饰器": "a.md,目录/c.md", // 3 个字符以上走 FTS
"检索": "b.md,目录/c.md", // 不足 3 个字符走 LIKE
"codemirror": "a.md", // 不区分大小写
"装饰器 检索": "目录/c.md", // 多个词同时满足
"目录": "目录/c.md", // 命中路径
"不存在的词": "",
"100%": "",
`"引号"`: "",
" ": "",
}
for q, want := range cases {
if got := f.paths(q); got != want && !(strings.Contains(want, ",") && sameSet(got, want)) {
t.Errorf("搜索 %q = %q,期望 %q", q, got, want)
}
}
}
func sameSet(a, b string) bool {
as, bs := strings.Split(a, ","), strings.Split(b, ",")
if len(as) != len(bs) {
return false
}
seen := map[string]bool{}
for _, s := range as {
seen[s] = true
}
for _, s := range bs {
if !seen[s] {
return false
}
}
return true
}
func TestSearchRanksNameMatchFirstAndReturnsSnippet(t *testing.T) {
f := newFixture(t)
f.write("其他.md", "第一行\n这里提到部署流程\n")
f.write("部署流程.md", "正文无关\n")
f.write("长行.md", strings.Repeat("前", 300)+"关键字"+strings.Repeat("后", 300))
f.sync()
rs, err := f.idx.Search("部署流程", 50)
if err != nil {
t.Fatal(err)
}
if len(rs) != 2 || rs[0].Path != "部署流程.md" || rs[0].Line != 0 {
t.Fatalf("文件名命中应排第一且无正文摘录: %+v", rs)
}
if rs[1].Line != 2 || rs[1].Snippet != "这里提到部署流程" {
t.Fatalf("摘录错误: %+v", rs[1])
}
rs, _ = f.idx.Search("关键字", 50)
s := rs[0].Snippet
if !strings.HasPrefix(s, "…") || !strings.HasSuffix(s, "…") || !strings.Contains(s, "关键字") || len([]rune(s)) > snippetRunes+2 {
t.Fatalf("长行摘录应截取匹配附近: %q", s)
}
}
func TestSyncTracksChanges(t *testing.T) {
f := newFixture(t)
f.write("a.md", "旧内容")
f.write("b.md", "保留")
f.sync()
f.write("a.md", "新内容")
if err := os.Remove(filepath.Join(f.v.Root(), "b.md")); err != nil {
t.Fatal(err)
}
f.write("c.md", "新增")
f.sync()
for q, want := range map[string]string{"旧内容": "", "新内容": "a.md", "保留": "", "新增": "c.md"} {
if got := f.paths(q); got != want {
t.Errorf("搜索 %q = %q,期望 %q", q, got, want)
}
}
// 重启后沿用已有索引,只处理变化
f.reopen()
if len(f.idx.known) != 2 {
t.Fatalf("重启后应载入已索引笔记: %v", f.idx.known)
}
f.sync()
if got := f.paths("新内容"); got != "a.md" {
t.Errorf("重启后搜索 = %q", got)
}
}
+18 -2
View File
@@ -34,20 +34,23 @@ type Watcher struct {
fw *fsnotify.Watcher
snap map[string]fileState
watched map[string]bool
onScan func(notes map[string]string)
}
func Start(ctx context.Context, v *vault.Vault, hub *events.Hub) error {
// Start 开始监听。onScan 在首次扫描及之后每次有变化的扫描后调用,参数为全部笔记的路径 -> rev(用于更新搜索索引)
func Start(ctx context.Context, v *vault.Vault, hub *events.Hub, onScan func(notes map[string]string)) error {
fw, err := fsnotify.NewWatcher()
if err != nil {
return err
}
w := &Watcher{v: v, hub: hub, fw: fw, snap: map[string]fileState{}, watched: map[string]bool{}}
w := &Watcher{v: v, hub: hub, fw: fw, snap: map[string]fileState{}, watched: map[string]bool{}, onScan: onScan}
snap, err := w.scan()
if err != nil {
fw.Close()
return err
}
w.snap = snap
w.onScan(notes(snap))
go w.loop(ctx)
return nil
}
@@ -90,9 +93,22 @@ func (w *Watcher) rescan() {
}
evs := diff(w.snap, next)
w.snap = next
if len(evs) > 0 {
w.onScan(notes(next))
}
w.hub.Broadcast(evs...)
}
func notes(snap map[string]fileState) map[string]string {
out := make(map[string]string, len(snap))
for p, st := range snap {
if !st.dir {
out[p] = st.rev
}
}
return out
}
// scan 读取当前目录树;内容版本只对新出现或大小/修改时间变化的文件重新计算
func (w *Watcher) scan() (map[string]fileState, error) {
entries, err := w.v.Tree()