feat(knowledge): index note relationships and maintain moved links
This commit is contained in:
eezd committed
1 parent
d34d1f2183
commit
a923c2dfb9
19 files changed
+545
-79
No files matched your search
+37
-25
@@ -7,7 +7,6 @@ import (
|
||||
"html"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
@@ -48,60 +47,73 @@ func (v *Vault) ReplaceImage(rel, filename string, r io.Reader, limit int64) err
|
||||
return atomicWrite(p, data)
|
||||
}
|
||||
|
||||
type imageLinkChange struct {
|
||||
type linkChange struct {
|
||||
path string
|
||||
before, after []byte
|
||||
}
|
||||
|
||||
// 调用方持有写锁；所有原始内容留到移动成功，以便复合操作失败时回滚。
|
||||
func (v *Vault) imageLinkChanges(from, to string) ([]imageLinkChange, error) {
|
||||
// 调用方持有写锁；先计算全部改写，移动成功后再写入，出错时保留回滚依据。
|
||||
func (v *Vault) moveLinkChanges(from, to string, directory bool, ownAssets bool) ([]linkChange, error) {
|
||||
entries, err := v.Tree()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var changes []imageLinkChange
|
||||
for _, e := range entries {
|
||||
if e.Dir || !IsNote(e.Path) {
|
||||
var changes []linkChange
|
||||
for _, entry := range entries {
|
||||
if entry.Dir || !IsNote(entry.Path) {
|
||||
continue
|
||||
}
|
||||
p := filepath.Join(v.root, filepath.FromSlash(e.Path))
|
||||
before, err := os.ReadFile(p)
|
||||
before, _, err := v.Read(entry.Path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
after, err := rewriteImageLinks(before, e.Path, from, to)
|
||||
next := movedPath(entry.Path, from, to, directory)
|
||||
after, err := rewriteMovedLinks(before, entry.Path, next, from, to, directory, ownAssets)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ownAssets && entry.Path == from {
|
||||
after = RewriteAssetLinks(after, strings.TrimSuffix(path.Base(from), NoteExt), strings.TrimSuffix(path.Base(to), NoteExt))
|
||||
}
|
||||
if !bytes.Equal(before, after) {
|
||||
changes = append(changes, imageLinkChange{p, before, after})
|
||||
changes = append(changes, linkChange{filepath.Join(v.root, filepath.FromSlash(next)), before, after})
|
||||
}
|
||||
}
|
||||
return changes, nil
|
||||
}
|
||||
|
||||
func rewriteImageLinks(content []byte, note, from, to string) ([]byte, error) {
|
||||
rel, err := filepath.Rel(filepath.FromSlash(path.Dir(note)), filepath.FromSlash(to))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
func movedPath(p, from, to string, directory bool) string {
|
||||
if p == from {
|
||||
return to
|
||||
}
|
||||
spans := importLinkSpans(content)
|
||||
if directory && strings.HasPrefix(p, from+"/") {
|
||||
return to + strings.TrimPrefix(p, from)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func rewriteMovedLinks(content []byte, oldNote, newNote, from, to string, directory, ownAssets bool) ([]byte, error) {
|
||||
var out []byte
|
||||
last := 0
|
||||
for _, span := range spans {
|
||||
for _, span := range importLinkSpans(content) {
|
||||
if span.start < last {
|
||||
continue
|
||||
}
|
||||
target := string(content[span.start:span.end])
|
||||
if span.html {
|
||||
target = html.UnescapeString(target)
|
||||
} else {
|
||||
target = markdownEscape.ReplaceAllString(target, "$1")
|
||||
}
|
||||
u, err := url.Parse(target)
|
||||
if err != nil || u.IsAbs() || u.Host != "" || strings.HasPrefix(target, "/") || u.Path == "" || path.Clean(path.Join(path.Dir(note), u.Path)) != from {
|
||||
target, u, ok := LocalTarget(oldNote, string(content[span.start:span.end]))
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
mapped := movedPath(target, from, to, directory)
|
||||
if ownAssets {
|
||||
mapped = movedPath(mapped, AssetsDir(from), AssetsDir(to), true)
|
||||
}
|
||||
if mapped == target && path.Dir(oldNote) == path.Dir(newNote) {
|
||||
continue
|
||||
}
|
||||
rel, err := filepath.Rel(filepath.FromSlash(path.Dir(newNote)), filepath.FromSlash(mapped))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
replacement := EscapeLinkPath(filepath.ToSlash(rel))
|
||||
if u.RawQuery != "" {
|
||||
replacement += "?" + u.RawQuery
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
package vault
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"html"
|
||||
"net/url"
|
||||
"path"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/yuin/goldmark/v2/ast"
|
||||
"github.com/yuin/goldmark/v2/parser"
|
||||
"go.yaml.in/yaml/v3"
|
||||
)
|
||||
|
||||
type Reference struct {
|
||||
Target string `json:"target"`
|
||||
Line int `json:"line"`
|
||||
}
|
||||
|
||||
type Metadata struct {
|
||||
Title string `json:"title"`
|
||||
Tags []string `json:"tags"`
|
||||
Links []Reference `json:"links"`
|
||||
Assets []Reference `json:"assets"`
|
||||
Warnings []string `json:"warnings"`
|
||||
}
|
||||
|
||||
var inlineTag = regexp.MustCompile(`#[\p{L}\p{N}_][\p{L}\p{N}_/-]*`)
|
||||
|
||||
// LocalTarget 在链接边界解码 Markdown 和 URL，拒绝远程地址、绝对路径及越界路径。
|
||||
func LocalTarget(note, raw string) (string, *url.URL, bool) {
|
||||
raw = html.UnescapeString(markdownEscape.ReplaceAllString(raw, "$1"))
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil || u.IsAbs() || u.Host != "" || strings.HasPrefix(raw, "/") || u.Path == "" {
|
||||
return "", nil, false
|
||||
}
|
||||
target := path.Clean(path.Join(path.Dir(note), u.Path))
|
||||
if _, err := CleanPath(target); err != nil {
|
||||
return "", nil, false
|
||||
}
|
||||
return target, u, true
|
||||
}
|
||||
|
||||
// Analyze 提取语义关系而不重新序列化正文；源位置保留为原文行号。
|
||||
func Analyze(note string, content []byte) Metadata {
|
||||
m := Metadata{Title: strings.TrimSuffix(path.Base(note), NoteExt), Tags: []string{}, Links: []Reference{}, Assets: []Reference{}, Warnings: []string{}}
|
||||
tags := map[string]bool{}
|
||||
addTag := func(s string) {
|
||||
s = strings.TrimPrefix(strings.TrimSpace(s), "#")
|
||||
if s != "" {
|
||||
tags[s] = true
|
||||
}
|
||||
}
|
||||
source := bytes.Clone(content)
|
||||
lines := bytes.SplitAfter(source, []byte("\n"))
|
||||
if len(lines) > 0 && strings.TrimSpace(string(lines[0])) == "---" {
|
||||
end, bodyEnd, offset := -1, 0, len(lines[0])
|
||||
for i := 1; i < len(lines); i++ {
|
||||
if strings.TrimSpace(string(lines[i])) == "---" {
|
||||
bodyEnd = offset
|
||||
end = offset + len(lines[i])
|
||||
break
|
||||
}
|
||||
offset += len(lines[i])
|
||||
}
|
||||
if end < 0 {
|
||||
m.Warnings = append(m.Warnings, "YAML 元数据未闭合")
|
||||
end = len(source)
|
||||
} else {
|
||||
var front struct {
|
||||
Tags yaml.Node `yaml:"tags"`
|
||||
Title string `yaml:"title"`
|
||||
}
|
||||
if err := yaml.Unmarshal(source[len(lines[0]):bodyEnd], &front); err != nil {
|
||||
m.Warnings = append(m.Warnings, fmt.Sprintf("YAML 元数据无效：%v", err))
|
||||
} else {
|
||||
if front.Title != "" {
|
||||
m.Title = front.Title
|
||||
}
|
||||
switch front.Tags.Kind {
|
||||
case 0:
|
||||
case yaml.ScalarNode:
|
||||
if front.Tags.Tag == "!!str" {
|
||||
for _, tag := range strings.FieldsFunc(front.Tags.Value, func(r rune) bool { return r == ',' || unicode.IsSpace(r) }) {
|
||||
addTag(tag)
|
||||
}
|
||||
} else {
|
||||
m.Warnings = append(m.Warnings, "YAML tags 必须是字符串或字符串列表")
|
||||
}
|
||||
case yaml.SequenceNode:
|
||||
for _, tag := range front.Tags.Content {
|
||||
if tag.Kind != yaml.ScalarNode || tag.Tag != "!!str" {
|
||||
m.Warnings = append(m.Warnings, "YAML tags 必须是字符串列表")
|
||||
break
|
||||
}
|
||||
addTag(tag.Value)
|
||||
}
|
||||
default:
|
||||
m.Warnings = append(m.Warnings, "YAML tags 必须是字符串或字符串列表")
|
||||
}
|
||||
}
|
||||
}
|
||||
for i := 0; i < end; i++ {
|
||||
if source[i] != '\n' && source[i] != '\r' {
|
||||
source[i] = ' '
|
||||
}
|
||||
}
|
||||
}
|
||||
root := parser.New().Parse(source)
|
||||
_ = ast.Walk(root, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
|
||||
if !entering {
|
||||
return ast.WalkContinue, nil
|
||||
}
|
||||
addRef := func(raw string, pos int) {
|
||||
target, _, ok := LocalTarget(note, raw)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
ref := Reference{target, bytes.Count(source[:max(0, pos)], []byte("\n")) + 1}
|
||||
if IsNote(target) {
|
||||
m.Links = append(m.Links, ref)
|
||||
} else {
|
||||
m.Assets = append(m.Assets, ref)
|
||||
}
|
||||
}
|
||||
switch node := n.(type) {
|
||||
case *ast.CodeBlock, *ast.CodeSpan, *ast.HTMLBlock, *ast.RawHTML, *ast.LinkReferenceDefinition:
|
||||
return ast.WalkSkipChildren, nil
|
||||
case *ast.Link:
|
||||
addRef(node.Destination.Value(source), node.Pos())
|
||||
return ast.WalkSkipChildren, nil
|
||||
case *ast.Image:
|
||||
addRef(node.Destination.Value(source), node.Pos())
|
||||
return ast.WalkSkipChildren, nil
|
||||
case *ast.Text:
|
||||
value := node.Value.Value(source)
|
||||
for _, loc := range inlineTag.FindAllStringIndex(value, -1) {
|
||||
if loc[0] > 0 {
|
||||
prev, _ := utf8.DecodeLastRuneInString(value[:loc[0]])
|
||||
if unicode.IsLetter(prev) || unicode.IsNumber(prev) || prev == '/' || prev == '\\' || prev == '&' {
|
||||
continue
|
||||
}
|
||||
}
|
||||
addTag(value[loc[0]:loc[1]])
|
||||
}
|
||||
}
|
||||
return ast.WalkContinue, nil
|
||||
})
|
||||
// HTML 资源也可能被历史导入的笔记引用；清理不能漏掉它们。
|
||||
for _, span := range importLinkSpans(content) {
|
||||
if !span.html {
|
||||
continue
|
||||
}
|
||||
target, _, ok := LocalTarget(note, string(content[span.start:span.end]))
|
||||
if ok {
|
||||
m.Assets = append(m.Assets, Reference{target, bytes.Count(content[:span.start], []byte("\n")) + 1})
|
||||
}
|
||||
}
|
||||
for tag := range tags {
|
||||
m.Tags = append(m.Tags, tag)
|
||||
}
|
||||
sort.Strings(m.Tags)
|
||||
return m
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package vault
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAnalyzeRelationshipsAndTags(t *testing.T) {
|
||||
content := "---\ntags: [知识, daily]\n---\n# 标题\n正文 #读书 和 #tag/nested\n[笔记](../目标.md#章节)  [引用][ref]\n\n[ref]: other.md\n\n`#code [伪](bad.md)`\n```\n#not-tag\n[fake](fake.md)\n```\n"
|
||||
m := Analyze("dir/a.md", []byte(content))
|
||||
if !reflect.DeepEqual(m.Tags, []string{"daily", "tag/nested", "知识", "读书"}) {
|
||||
t.Fatalf("标签不符：%v", m.Tags)
|
||||
}
|
||||
if len(m.Links) != 2 || m.Links[0].Target != "目标.md" || m.Links[1].Target != "dir/other.md" || m.Links[0].Line != 6 {
|
||||
t.Fatalf("链接不符：%+v", m.Links)
|
||||
}
|
||||
if len(m.Assets) != 1 || m.Assets[0].Target != "dir/a.assets/p.png" {
|
||||
t.Fatalf("图片引用不符：%+v", m.Assets)
|
||||
}
|
||||
if len(m.Warnings) != 0 {
|
||||
t.Fatal(m.Warnings)
|
||||
}
|
||||
for _, content := range []string{"---\ntags: [broken\n---\n", "---\ntags: {key: value}\n---", "---\ntags: 12\n---", "---\ntags: x"} {
|
||||
if len(Analyze("x.md", []byte(content)).Warnings) == 0 {
|
||||
t.Fatalf("无效元数据未报告：%s", content)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMoveMaintainsIncomingAndOutgoingLinks(t *testing.T) {
|
||||
v := newVault(t)
|
||||
for p, body := range map[string]string{
|
||||
"dir/a.md": "[外部](../other.md) [同级](b.md#section) \n`[不改](b.md)`\n",
|
||||
"dir/b.md": "[入链](a.md?q=1#section)", "other.md": "[入链](dir/a.md)",
|
||||
} {
|
||||
if _, err := v.Write(p, []byte(body), Precondition{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := v.SaveAsset("dir/a.md", "p.png", strings.NewReader("image"), 100); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := v.Move("dir/a.md", "new/renamed.md", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, _, _ := v.Read("new/renamed.md")
|
||||
if !strings.Contains(string(body), "../dir/b.md#section") || !strings.Contains(string(body), "renamed.assets/p.png") || !strings.Contains(string(body), "`[不改](b.md)`") {
|
||||
t.Fatalf("出链错误：%s", body)
|
||||
}
|
||||
body, _, _ = v.Read("dir/b.md")
|
||||
if string(body) != "[入链](../new/renamed.md?q=1#section)" {
|
||||
t.Fatalf("入链错误：%s", body)
|
||||
}
|
||||
if _, err := v.Move("new", "deep/folder", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, _, _ = v.Read("deep/folder/renamed.md")
|
||||
if !strings.Contains(string(body), "../../other.md") {
|
||||
t.Fatalf("目录出链错误：%s", body)
|
||||
}
|
||||
original, _, _ := v.Read("other.md")
|
||||
failure := errors.New("提交失败")
|
||||
if _, err := v.Move("deep/folder", "rollback", func() error { return failure }); !errors.Is(err, failure) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, _, _ = v.Read("other.md")
|
||||
if string(body) != string(original) {
|
||||
t.Fatalf("入链未回滚：%s", body)
|
||||
}
|
||||
if _, _, err := v.Read("deep/folder/renamed.md"); err != nil {
|
||||
t.Fatal("移动主体未回滚", err)
|
||||
}
|
||||
}
|
||||
+9
-29
@@ -440,13 +440,10 @@ func (v *Vault) Move(from, to string, commit func() error) (rewritten bool, err
|
||||
}
|
||||
}
|
||||
}
|
||||
// 图片改名影响引用它的笔记：先读取全部待改正文，失败时不能先移动图片。
|
||||
var links []imageLinkChange
|
||||
if isImage {
|
||||
links, err = v.imageLinkChanges(from, to)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
// 笔记、目录和图片的移动都可能改变跨文件相对链接。
|
||||
links, err := v.moveLinkChanges(from, to, info.IsDir(), srcAssets != "")
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
|
||||
return false, err
|
||||
@@ -456,19 +453,13 @@ func (v *Vault) Move(from, to string, commit func() error) (rewritten bool, err
|
||||
}
|
||||
assetsMoved := false
|
||||
writtenLinks := 0
|
||||
var originalNote []byte
|
||||
defer func() {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
if rewritten {
|
||||
if rollback := atomicWrite(dst, originalNote); rollback != nil {
|
||||
err = errors.Join(err, fmt.Errorf("回滚笔记正文失败: %w", rollback))
|
||||
}
|
||||
}
|
||||
for _, change := range links[:writtenLinks] {
|
||||
if rollback := atomicWrite(change.path, change.before); rollback != nil {
|
||||
err = errors.Join(err, fmt.Errorf("回滚图片引用失败: %w", rollback))
|
||||
err = errors.Join(err, fmt.Errorf("回滚笔记引用失败: %w", rollback))
|
||||
}
|
||||
}
|
||||
if assetsMoved {
|
||||
@@ -485,27 +476,16 @@ func (v *Vault) Move(from, to string, commit func() error) (rewritten bool, err
|
||||
return false, err
|
||||
}
|
||||
writtenLinks++
|
||||
if change.path == dst || info.IsDir() && strings.HasPrefix(change.path, dst+string(filepath.Separator)) {
|
||||
rewritten = true
|
||||
}
|
||||
}
|
||||
if srcAssets != "" {
|
||||
if err := os.Rename(srcAssets, dstAssets); err != nil {
|
||||
return false, fmt.Errorf("移动图片目录失败: %w", err)
|
||||
}
|
||||
assetsMoved = true
|
||||
oldBase := strings.TrimSuffix(path.Base(from), NoteExt)
|
||||
newBase := strings.TrimSuffix(path.Base(to), NoteExt)
|
||||
if oldBase != newBase {
|
||||
data, err := os.ReadFile(dst)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if out := RewriteAssetLinks(data, oldBase, newBase); !bytes.Equal(out, data) {
|
||||
if err = atomicWrite(dst, out); err != nil {
|
||||
return false, err
|
||||
}
|
||||
originalNote = data
|
||||
rewritten = true
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
if commit != nil {
|
||||
err = commit()
|
||||
|
||||
Reference in new issue
Block a user