Add full-text search
- Index notes in a separate, rebuildable SQLite database (index.db) using FTS5 trigram, synced from watcher snapshots so external edits are picked up; terms under 3 characters fall back to LIKE - GET /api/v1/search?q= returns matching notes with the first matching line and a snippet; path matches rank first - Sidebar search panel (Ctrl+Shift+F) with highlighted terms and keyboard navigation; opening a result jumps to and selects the match - Restore scroll position on tab switch with scroll snapshots; restoring scrollTop drifted by a few lines after line heights were re-estimated
This commit is contained in:
1 parent
a11e0ec498
commit
55f0ee174e
13 files changed
+792
-20
No files matched your search
+16
-2
@@ -16,6 +16,7 @@ import (
|
||||
|
||||
"rawmark/internal/auth"
|
||||
"rawmark/internal/events"
|
||||
"rawmark/internal/search"
|
||||
"rawmark/internal/vault"
|
||||
)
|
||||
|
||||
@@ -30,12 +31,13 @@ type Server struct {
|
||||
vault *vault.Vault
|
||||
auth *auth.Manager
|
||||
hub *events.Hub
|
||||
index *search.Index
|
||||
// OnSetup 在首次初始化完成后调用(用于删除启动码文件)
|
||||
OnSetup func()
|
||||
}
|
||||
|
||||
func New(v *vault.Vault, a *auth.Manager, hub *events.Hub) *Server {
|
||||
return &Server{vault: v, auth: a, hub: hub}
|
||||
func New(v *vault.Vault, a *auth.Manager, hub *events.Hub, index *search.Index) *Server {
|
||||
return &Server{vault: v, auth: a, hub: hub, index: index}
|
||||
}
|
||||
|
||||
func (s *Server) Handler(static fs.FS) http.Handler {
|
||||
@@ -59,6 +61,7 @@ func (s *Server) Handler(static fs.FS) http.Handler {
|
||||
"POST /api/v1/trash/{id}/restore": s.trashRestore,
|
||||
"DELETE /api/v1/trash/{id}": s.trashPurge,
|
||||
"GET /api/v1/events": s.events,
|
||||
"GET /api/v1/search": s.search,
|
||||
}
|
||||
for pattern, h := range protected {
|
||||
mux.Handle(pattern, s.requireAuth(h))
|
||||
@@ -211,6 +214,17 @@ func (s *Server) tree(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, 200, map[string]any{"entries": entries})
|
||||
}
|
||||
|
||||
const searchLimit = 50
|
||||
|
||||
func (s *Server) search(w http.ResponseWriter, r *http.Request) {
|
||||
results, err := s.index.Search(r.URL.Query().Get("q"), searchLimit)
|
||||
if err != nil {
|
||||
internalError(w, r, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, map[string]any{"results": results})
|
||||
}
|
||||
|
||||
func (s *Server) readFile(w http.ResponseWriter, r *http.Request) {
|
||||
data, rev, err := s.vault.Read(r.PathValue("path"))
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,310 @@
|
||||
// Package search 维护笔记的全文索引。索引存放在独立的数据库文件中,内容完全由仓库文件派生,
|
||||
// 删除后会在下次启动时重建;结构变化时也直接重建而不做迁移。
|
||||
package search
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"sync"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
|
||||
"rawmark/internal/vault"
|
||||
)
|
||||
|
||||
// 修改表结构时递增,旧索引会被丢弃重建
|
||||
const schemaVersion = 1
|
||||
|
||||
const schema = `
|
||||
CREATE TABLE docs (
|
||||
id INTEGER PRIMARY KEY,
|
||||
path TEXT NOT NULL UNIQUE,
|
||||
rev TEXT NOT NULL
|
||||
);
|
||||
-- trigram 分词不依赖空格断词,中文可按任意片段检索;不足 3 个字符的词改用 LIKE
|
||||
CREATE VIRTUAL TABLE docs_fts USING fts5(path, body, tokenize = 'trigram');
|
||||
`
|
||||
|
||||
const (
|
||||
maxTerms = 8
|
||||
snippetRunes = 160
|
||||
snippetBefore = 40
|
||||
)
|
||||
|
||||
type Result struct {
|
||||
Path string `json:"path"`
|
||||
// 第一处匹配所在行(从 1 开始)与该行的摘录;只有文件名匹配时 Line 为 0
|
||||
Line int `json:"line"`
|
||||
Snippet string `json:"snippet"`
|
||||
}
|
||||
|
||||
type doc struct {
|
||||
id int64
|
||||
rev string
|
||||
}
|
||||
|
||||
type Index struct {
|
||||
db *sql.DB
|
||||
v *vault.Vault
|
||||
|
||||
mu sync.Mutex
|
||||
pending map[string]string // 待同步的最新快照(路径 -> rev)
|
||||
wake chan struct{}
|
||||
|
||||
known map[string]doc // 已索引的笔记,仅由同步协程访问
|
||||
}
|
||||
|
||||
func Open(path string, v *vault.Vault) (*Index, error) {
|
||||
db, err := sql.Open("sqlite", "file:"+path+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
x := &Index{db: db, v: v, wake: make(chan struct{}, 1), known: map[string]doc{}}
|
||||
if err := x.init(); err != nil {
|
||||
db.Close()
|
||||
return nil, fmt.Errorf("初始化搜索索引失败: %w", err)
|
||||
}
|
||||
return x, nil
|
||||
}
|
||||
|
||||
func (x *Index) init() error {
|
||||
var version int
|
||||
if err := x.db.QueryRow(`PRAGMA user_version`).Scan(&version); err != nil {
|
||||
return err
|
||||
}
|
||||
if version != schemaVersion {
|
||||
stmts := []string{`DROP TABLE IF EXISTS docs`, `DROP TABLE IF EXISTS docs_fts`, schema, fmt.Sprintf(`PRAGMA user_version = %d`, schemaVersion)}
|
||||
for _, s := range stmts {
|
||||
if _, err := x.db.Exec(s); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
rows, err := x.db.Query(`SELECT id, path, rev FROM docs`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var d doc
|
||||
var p string
|
||||
if err := rows.Scan(&d.id, &p, &d.rev); err != nil {
|
||||
return err
|
||||
}
|
||||
x.known[p] = d
|
||||
}
|
||||
return rows.Err()
|
||||
}
|
||||
|
||||
func (x *Index) Close() error { return x.db.Close() }
|
||||
|
||||
// Update 提交仓库的最新快照(笔记路径 -> rev),立即返回;同步在后台进行,多次提交只处理最新一次
|
||||
func (x *Index) Update(files map[string]string) {
|
||||
x.mu.Lock()
|
||||
x.pending = files
|
||||
x.mu.Unlock()
|
||||
select {
|
||||
case x.wake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// Run 执行后台同步,直到 ctx 结束
|
||||
func (x *Index) Run(ctx context.Context) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-x.wake:
|
||||
x.mu.Lock()
|
||||
files := x.pending
|
||||
x.mu.Unlock()
|
||||
if err := x.sync(files); err != nil {
|
||||
slog.Error("更新搜索索引失败", "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sync 让索引与快照一致:只读取新增或 rev 变化的笔记
|
||||
func (x *Index) sync(files map[string]string) error {
|
||||
tx, err := x.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
next := make(map[string]doc, len(x.known))
|
||||
for p, d := range x.known {
|
||||
next[p] = d
|
||||
}
|
||||
for p, d := range x.known {
|
||||
if _, ok := files[p]; !ok {
|
||||
if err := remove(tx, d.id); err != nil {
|
||||
return err
|
||||
}
|
||||
delete(next, p)
|
||||
}
|
||||
}
|
||||
for p, rev := range files {
|
||||
old, ok := x.known[p]
|
||||
if ok && old.rev == rev {
|
||||
continue
|
||||
}
|
||||
content, rev, err := x.v.Read(p)
|
||||
if errors.Is(err, vault.ErrNotFound) {
|
||||
continue // 快照之后被删除,下一次快照会处理
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ok {
|
||||
if err := remove(tx, old.id); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
res, err := tx.Exec(`INSERT INTO docs (path, rev) VALUES (?, ?)`, p, rev)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
id, err := res.LastInsertId()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(`INSERT INTO docs_fts (rowid, path, body) VALUES (?, ?, ?)`, id, p, string(content)); err != nil {
|
||||
return err
|
||||
}
|
||||
next[p] = doc{id: id, rev: rev}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
x.known = next
|
||||
return nil
|
||||
}
|
||||
|
||||
func remove(tx *sql.Tx, id int64) error {
|
||||
if _, err := tx.Exec(`DELETE FROM docs WHERE id = ?`, id); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := tx.Exec(`DELETE FROM docs_fts WHERE rowid = ?`, id)
|
||||
return err
|
||||
}
|
||||
|
||||
// Search 查找同时包含所有词(空白分隔,不区分大小写)的笔记,词可出现在路径或正文中。
|
||||
// 路径(文件名或目录名)命中第一个词的排在前面,其余按相关度排序
|
||||
func (x *Index) Search(query string, limit int) ([]Result, error) {
|
||||
terms := strings.Fields(query)
|
||||
if len(terms) == 0 {
|
||||
return []Result{}, nil
|
||||
}
|
||||
if len(terms) > maxTerms {
|
||||
terms = terms[:maxTerms]
|
||||
}
|
||||
var where, phrases []string
|
||||
var args []any
|
||||
for _, t := range terms {
|
||||
if utf8.RuneCountInString(t) >= 3 {
|
||||
phrases = append(phrases, `"`+strings.ReplaceAll(t, `"`, `""`)+`"`)
|
||||
} else {
|
||||
like := "%" + escapeLike(t) + "%"
|
||||
where = append(where, `(path LIKE ? ESCAPE '\' OR body LIKE ? ESCAPE '\')`)
|
||||
args = append(args, like, like)
|
||||
}
|
||||
}
|
||||
order := "path"
|
||||
if len(phrases) > 0 {
|
||||
where = append(where, `docs_fts MATCH ?`)
|
||||
args = append(args, strings.Join(phrases, " AND "))
|
||||
order = "rank"
|
||||
}
|
||||
name := "%" + escapeLike(terms[0]) + "%"
|
||||
q := `SELECT path, body FROM docs_fts WHERE ` + strings.Join(where, " AND ") +
|
||||
` ORDER BY path LIKE ? ESCAPE '\' DESC, ` + order + ` LIMIT ?`
|
||||
args = append(args, name, limit)
|
||||
rows, err := x.db.Query(q, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
results := []Result{}
|
||||
for rows.Next() {
|
||||
var r Result
|
||||
var body string
|
||||
if err := rows.Scan(&r.Path, &body); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.Line, r.Snippet = snippet(body, terms)
|
||||
results = append(results, r)
|
||||
}
|
||||
return results, rows.Err()
|
||||
}
|
||||
|
||||
func escapeLike(s string) string {
|
||||
return strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(s)
|
||||
}
|
||||
|
||||
// snippet 返回正文中第一处匹配所在的行号与摘录(过长的行截取匹配附近的一段)
|
||||
func snippet(body string, terms []string) (int, string) {
|
||||
folded := make([][]rune, len(terms))
|
||||
for i, t := range terms {
|
||||
folded[i] = fold([]rune(t))
|
||||
}
|
||||
for n, line := range strings.Split(body, "\n") {
|
||||
runes := []rune(strings.TrimRight(line, "\r"))
|
||||
lower := fold(runes)
|
||||
at := -1
|
||||
for _, t := range folded {
|
||||
if i := indexRunes(lower, t); i >= 0 && (at < 0 || i < at) {
|
||||
at = i
|
||||
}
|
||||
}
|
||||
if at < 0 {
|
||||
continue
|
||||
}
|
||||
start := 0
|
||||
if len(runes) > snippetRunes {
|
||||
start = max(0, min(at-snippetBefore, len(runes)-snippetRunes))
|
||||
}
|
||||
end := min(len(runes), start+snippetRunes)
|
||||
text := strings.TrimSpace(string(runes[start:end]))
|
||||
if start > 0 {
|
||||
text = "…" + text
|
||||
}
|
||||
if end < len(runes) {
|
||||
text += "…"
|
||||
}
|
||||
return n + 1, text
|
||||
}
|
||||
return 0, ""
|
||||
}
|
||||
|
||||
// fold 逐字符转小写,保持字符数不变,便于在原文中按位置截取
|
||||
func fold(rs []rune) []rune {
|
||||
out := make([]rune, len(rs))
|
||||
for i, r := range rs {
|
||||
out[i] = unicode.ToLower(r)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func indexRunes(s, sub []rune) int {
|
||||
for i := 0; i+len(sub) <= len(s); i++ {
|
||||
match := true
|
||||
for j := range sub {
|
||||
if s[i+j] != sub[j] {
|
||||
match = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if match {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
package search
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"rawmark/internal/vault"
|
||||
)
|
||||
|
||||
type fixture struct {
|
||||
t *testing.T
|
||||
v *vault.Vault
|
||||
idx *Index
|
||||
db string
|
||||
}
|
||||
|
||||
func newFixture(t *testing.T) *fixture {
|
||||
t.Helper()
|
||||
v, err := vault.Open(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := &fixture{t: t, v: v, db: filepath.Join(t.TempDir(), "index.db")}
|
||||
f.reopen()
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *fixture) reopen() {
|
||||
if f.idx != nil {
|
||||
f.idx.Close()
|
||||
}
|
||||
idx, err := Open(f.db, f.v)
|
||||
if err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
f.idx = idx
|
||||
f.t.Cleanup(func() { idx.Close() })
|
||||
}
|
||||
|
||||
func (f *fixture) write(p, content string) {
|
||||
abs := filepath.Join(f.v.Root(), p)
|
||||
if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(abs, []byte(content), 0o644); err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// sync 模拟监听器提交快照
|
||||
func (f *fixture) sync() {
|
||||
f.t.Helper()
|
||||
entries, err := f.v.Tree()
|
||||
if err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
files := map[string]string{}
|
||||
for _, e := range entries {
|
||||
if !e.Dir {
|
||||
data, _, err := f.v.Read(e.Path)
|
||||
if err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
files[e.Path] = vault.Rev(data)
|
||||
}
|
||||
}
|
||||
if err := f.idx.sync(files); err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fixture) paths(q string) string {
|
||||
f.t.Helper()
|
||||
rs, err := f.idx.Search(q, 50)
|
||||
if err != nil {
|
||||
f.t.Fatalf("搜索 %q: %v", q, err)
|
||||
}
|
||||
var out []string
|
||||
for _, r := range rs {
|
||||
out = append(out, r.Path)
|
||||
}
|
||||
return strings.Join(out, ",")
|
||||
}
|
||||
|
||||
func TestSearchTerms(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
f.write("a.md", "# 标题\n\n今天学习 CodeMirror 的装饰器\n")
|
||||
f.write("b.md", "第二篇笔记,讲 SQLite 全文检索\n")
|
||||
f.write("目录/c.md", "装饰器和检索都提到\n")
|
||||
f.sync()
|
||||
|
||||
cases := map[string]string{
|
||||
"装饰器": "a.md,目录/c.md", // 3 个字符以上走 FTS
|
||||
"检索": "b.md,目录/c.md", // 不足 3 个字符走 LIKE
|
||||
"codemirror": "a.md", // 不区分大小写
|
||||
"装饰器 检索": "目录/c.md", // 多个词同时满足
|
||||
"目录": "目录/c.md", // 命中路径
|
||||
"不存在的词": "",
|
||||
"100%": "",
|
||||
`"引号"`: "",
|
||||
" ": "",
|
||||
}
|
||||
for q, want := range cases {
|
||||
if got := f.paths(q); got != want && !(strings.Contains(want, ",") && sameSet(got, want)) {
|
||||
t.Errorf("搜索 %q = %q,期望 %q", q, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func sameSet(a, b string) bool {
|
||||
as, bs := strings.Split(a, ","), strings.Split(b, ",")
|
||||
if len(as) != len(bs) {
|
||||
return false
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, s := range as {
|
||||
seen[s] = true
|
||||
}
|
||||
for _, s := range bs {
|
||||
if !seen[s] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func TestSearchRanksNameMatchFirstAndReturnsSnippet(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
f.write("其他.md", "第一行\n这里提到部署流程\n")
|
||||
f.write("部署流程.md", "正文无关\n")
|
||||
f.write("长行.md", strings.Repeat("前", 300)+"关键字"+strings.Repeat("后", 300))
|
||||
f.sync()
|
||||
|
||||
rs, err := f.idx.Search("部署流程", 50)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rs) != 2 || rs[0].Path != "部署流程.md" || rs[0].Line != 0 {
|
||||
t.Fatalf("文件名命中应排第一且无正文摘录: %+v", rs)
|
||||
}
|
||||
if rs[1].Line != 2 || rs[1].Snippet != "这里提到部署流程" {
|
||||
t.Fatalf("摘录错误: %+v", rs[1])
|
||||
}
|
||||
|
||||
rs, _ = f.idx.Search("关键字", 50)
|
||||
s := rs[0].Snippet
|
||||
if !strings.HasPrefix(s, "…") || !strings.HasSuffix(s, "…") || !strings.Contains(s, "关键字") || len([]rune(s)) > snippetRunes+2 {
|
||||
t.Fatalf("长行摘录应截取匹配附近: %q", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncTracksChanges(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
f.write("a.md", "旧内容")
|
||||
f.write("b.md", "保留")
|
||||
f.sync()
|
||||
|
||||
f.write("a.md", "新内容")
|
||||
if err := os.Remove(filepath.Join(f.v.Root(), "b.md")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.write("c.md", "新增")
|
||||
f.sync()
|
||||
for q, want := range map[string]string{"旧内容": "", "新内容": "a.md", "保留": "", "新增": "c.md"} {
|
||||
if got := f.paths(q); got != want {
|
||||
t.Errorf("搜索 %q = %q,期望 %q", q, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// 重启后沿用已有索引,只处理变化
|
||||
f.reopen()
|
||||
if len(f.idx.known) != 2 {
|
||||
t.Fatalf("重启后应载入已索引笔记: %v", f.idx.known)
|
||||
}
|
||||
f.sync()
|
||||
if got := f.paths("新内容"); got != "a.md" {
|
||||
t.Errorf("重启后搜索 = %q", got)
|
||||
}
|
||||
}
|
||||
+18
-2
@@ -34,20 +34,23 @@ type Watcher struct {
|
||||
fw *fsnotify.Watcher
|
||||
snap map[string]fileState
|
||||
watched map[string]bool
|
||||
onScan func(notes map[string]string)
|
||||
}
|
||||
|
||||
func Start(ctx context.Context, v *vault.Vault, hub *events.Hub) error {
|
||||
// Start 开始监听。onScan 在首次扫描及之后每次有变化的扫描后调用,参数为全部笔记的路径 -> rev(用于更新搜索索引)
|
||||
func Start(ctx context.Context, v *vault.Vault, hub *events.Hub, onScan func(notes map[string]string)) error {
|
||||
fw, err := fsnotify.NewWatcher()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w := &Watcher{v: v, hub: hub, fw: fw, snap: map[string]fileState{}, watched: map[string]bool{}}
|
||||
w := &Watcher{v: v, hub: hub, fw: fw, snap: map[string]fileState{}, watched: map[string]bool{}, onScan: onScan}
|
||||
snap, err := w.scan()
|
||||
if err != nil {
|
||||
fw.Close()
|
||||
return err
|
||||
}
|
||||
w.snap = snap
|
||||
w.onScan(notes(snap))
|
||||
go w.loop(ctx)
|
||||
return nil
|
||||
}
|
||||
@@ -90,9 +93,22 @@ func (w *Watcher) rescan() {
|
||||
}
|
||||
evs := diff(w.snap, next)
|
||||
w.snap = next
|
||||
if len(evs) > 0 {
|
||||
w.onScan(notes(next))
|
||||
}
|
||||
w.hub.Broadcast(evs...)
|
||||
}
|
||||
|
||||
func notes(snap map[string]fileState) map[string]string {
|
||||
out := make(map[string]string, len(snap))
|
||||
for p, st := range snap {
|
||||
if !st.dir {
|
||||
out[p] = st.rev
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// scan 读取当前目录树;内容版本只对新出现或大小/修改时间变化的文件重新计算
|
||||
func (w *Watcher) scan() (map[string]fileState, error) {
|
||||
entries, err := w.v.Tree()
|
||||
|
||||
Reference in new issue
Block a user