Add Markdown and ZIP notebook import
This commit is contained in:
1 parent
903a617439
commit
34d2acbbd5
18 files changed
+1166
-16
No files matched your search
@@ -0,0 +1,289 @@
|
||||
package vault
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"compress/flate"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"syscall"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxImportBytes = 100_000_000
|
||||
maxImportExpanded = 512 << 20
|
||||
maxImportEntries = 10_000
|
||||
importNoteLimit = 20 << 20
|
||||
importAssetLimit = 50 << 20
|
||||
)
|
||||
|
||||
var ErrInvalidImport = errors.New("导入内容无效")
|
||||
|
||||
// ImportFile 的读取范围是单个上传文件,ZIP 与普通 Markdown 共用这一输入边界。
|
||||
type ImportFile struct {
|
||||
Name string
|
||||
Reader io.ReaderAt
|
||||
Size int64
|
||||
}
|
||||
|
||||
type ImportResult struct {
|
||||
Paths []string `json:"paths"`
|
||||
Notes int `json:"notes"`
|
||||
Assets int `json:"assets"`
|
||||
Warnings []string `json:"warnings"`
|
||||
}
|
||||
|
||||
// Import 在隐藏目录内整理完整内容,提交前统一检查冲突,绝不覆盖已有文件。
|
||||
func (v *Vault) Import(ctx context.Context, dir, folder string, files []ImportFile) (result ImportResult, err error) {
|
||||
result = ImportResult{Paths: []string{}, Warnings: []string{}}
|
||||
if len(files) == 0 || len(files) > maxImportEntries {
|
||||
return result, fmt.Errorf("%w:请选择 Markdown 文件或一个 ZIP", ErrInvalidImport)
|
||||
}
|
||||
zipped := len(files) == 1 && strings.EqualFold(path.Ext(files[0].Name), ".zip")
|
||||
if zipped {
|
||||
if _, err := CleanPath(folder); err != nil || strings.Contains(folder, "/") || strings.HasSuffix(folder, AssetsExt) {
|
||||
return result, fmt.Errorf("%w:导入文件夹名称无效", ErrInvalidImport)
|
||||
}
|
||||
} else if folder != "" {
|
||||
return result, fmt.Errorf("%w:普通 Markdown 不使用外层文件夹", ErrInvalidImport)
|
||||
}
|
||||
if dir != "" {
|
||||
if _, err := CleanPath(dir); err != nil {
|
||||
return result, err
|
||||
}
|
||||
}
|
||||
var uploadBytes int64
|
||||
for _, f := range files {
|
||||
if f.Size < 0 || f.Size > MaxImportBytes-uploadBytes {
|
||||
return result, ErrTooLarge
|
||||
}
|
||||
uploadBytes += f.Size
|
||||
}
|
||||
stage, err := os.MkdirTemp(v.root, tmpPrefix+"import-")
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
defer func() {
|
||||
if cleanup := os.RemoveAll(stage); cleanup != nil {
|
||||
err = errors.Join(err, fmt.Errorf("清理导入临时目录失败:%w", cleanup))
|
||||
}
|
||||
}()
|
||||
staged := &Vault{root: stage}
|
||||
entries := map[string]struct{}{}
|
||||
var total int64
|
||||
var notes []string
|
||||
// 每个来源文件只写一次;目录与文件重名也会在暂存阶段报错。
|
||||
write := func(name string, isDir bool, reader io.Reader) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := CleanPath(name); err != nil || !utf8.ValidString(name) {
|
||||
return fmt.Errorf("%w:ZIP 含有非法路径", ErrInvalidImport)
|
||||
}
|
||||
if _, exists := entries[name]; exists {
|
||||
return fmt.Errorf("%w:存在重复路径", ErrInvalidImport)
|
||||
}
|
||||
entries[name] = struct{}{}
|
||||
dest := filepath.Join(stage, filepath.FromSlash(name))
|
||||
if isDir {
|
||||
err := os.MkdirAll(dest, 0o755)
|
||||
if errors.Is(err, fs.ErrExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
return fmt.Errorf("%w:文件与目录路径冲突", ErrInvalidImport)
|
||||
}
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
|
||||
if errors.Is(err, syscall.ENOTDIR) || errors.Is(err, fs.ErrExist) {
|
||||
return fmt.Errorf("%w:文件与目录路径冲突", ErrInvalidImport)
|
||||
}
|
||||
return err
|
||||
}
|
||||
f, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
|
||||
if err != nil {
|
||||
if errors.Is(err, fs.ErrExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
return fmt.Errorf("%w:文件与目录路径冲突", ErrInvalidImport)
|
||||
}
|
||||
return err
|
||||
}
|
||||
limit := int64(importAssetLimit)
|
||||
if IsNote(name) {
|
||||
limit = importNoteLimit
|
||||
notes = append(notes, name)
|
||||
}
|
||||
limit = min(limit, maxImportExpanded-total)
|
||||
n, copyErr := io.Copy(f, io.LimitReader(reader, limit+1))
|
||||
closeErr := f.Close()
|
||||
total += n
|
||||
if n > limit {
|
||||
return ErrTooLarge
|
||||
}
|
||||
return errors.Join(copyErr, closeErr)
|
||||
}
|
||||
if zipped {
|
||||
z, err := zip.NewReader(files[0].Reader, files[0].Size)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("%w:无法读取 ZIP:%w", ErrInvalidImport, err)
|
||||
}
|
||||
if len(z.File) > maxImportEntries {
|
||||
return result, ErrTooLarge
|
||||
}
|
||||
for _, f := range z.File {
|
||||
if !f.Mode().IsRegular() && !f.Mode().IsDir() {
|
||||
return result, fmt.Errorf("%w:ZIP 不允许符号链接或特殊文件", ErrInvalidImport)
|
||||
}
|
||||
name := strings.TrimSuffix(f.Name, "/")
|
||||
r, err := f.Open()
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("%w:无法读取 ZIP 条目:%w", ErrInvalidImport, err)
|
||||
}
|
||||
err = write(name, f.FileInfo().IsDir(), r)
|
||||
err = errors.Join(err, r.Close())
|
||||
if err != nil {
|
||||
var corrupt flate.CorruptInputError
|
||||
if errors.Is(err, zip.ErrChecksum) || errors.Is(err, io.ErrUnexpectedEOF) || errors.As(err, &corrupt) {
|
||||
err = fmt.Errorf("%w:ZIP 内容损坏:%w", ErrInvalidImport, err)
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for _, f := range files {
|
||||
if !IsNote(f.Name) || strings.Contains(f.Name, "/") {
|
||||
return result, fmt.Errorf("%w:只接受 .md 文件或单个 ZIP", ErrInvalidImport)
|
||||
}
|
||||
if err := write(f.Name, false, io.NewSectionReader(f.Reader, 0, f.Size)); err != nil {
|
||||
return result, err
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(notes) == 0 {
|
||||
return result, fmt.Errorf("%w:没有可导入的 Markdown 笔记", ErrInvalidImport)
|
||||
}
|
||||
sort.Strings(notes)
|
||||
for _, note := range notes {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return result, err
|
||||
}
|
||||
for _, seg := range strings.Split(path.Dir(note), "/") {
|
||||
if strings.HasSuffix(seg, AssetsExt) {
|
||||
return result, fmt.Errorf("%w:笔记不能放在 .assets 附件目录内", ErrInvalidImport)
|
||||
}
|
||||
}
|
||||
content, _, err := staged.Read(note)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
if !utf8.Valid(content) || bytes.IndexByte(content, 0) >= 0 {
|
||||
return result, fmt.Errorf("%w:Markdown 必须是 UTF-8 文本", ErrInvalidImport)
|
||||
}
|
||||
content = bytes.TrimPrefix(content, []byte{0xef, 0xbb, 0xbf})
|
||||
updated, warnings, err := staged.importLinks(ctx, note, content, &total)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.Warnings = append(result.Warnings, warnings...)
|
||||
if len(updated) > importNoteLimit || total+int64(len(updated)-len(content)) > maxImportExpanded {
|
||||
return result, ErrTooLarge
|
||||
}
|
||||
total += int64(len(updated) - len(content))
|
||||
if err := os.WriteFile(filepath.Join(stage, filepath.FromSlash(note)), updated, 0o644); err != nil {
|
||||
return result, err
|
||||
}
|
||||
}
|
||||
result.Notes = len(notes)
|
||||
// 附件数量包含原始附件及为各笔记生成的独立副本。
|
||||
count := 0
|
||||
err = filepath.WalkDir(stage, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
count++
|
||||
if count > maxImportEntries+1 {
|
||||
return ErrTooLarge
|
||||
}
|
||||
if !d.IsDir() && !IsNote(d.Name()) {
|
||||
result.Assets++
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
top, err := os.ReadDir(stage)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
targets := []string{}
|
||||
if zipped {
|
||||
targets = append(targets, path.Join(dir, folder))
|
||||
} else {
|
||||
for _, f := range top {
|
||||
targets = append(targets, path.Join(dir, f.Name()))
|
||||
}
|
||||
}
|
||||
v.mu.Lock()
|
||||
defer v.mu.Unlock()
|
||||
parent := v.root
|
||||
if dir != "" {
|
||||
parent, err = v.abs(dir)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
}
|
||||
info, err := os.Stat(parent)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return result, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return result, fmt.Errorf("%w:目标不是文件夹", ErrInvalidImport)
|
||||
}
|
||||
for _, target := range targets {
|
||||
dest, err := v.abs(target)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
if _, err := os.Lstat(dest); err == nil {
|
||||
return result, fmt.Errorf("%w:%s,请修改名称", ErrExists, target)
|
||||
} else if !errors.Is(err, fs.ErrNotExist) {
|
||||
return result, err
|
||||
}
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return result, err
|
||||
}
|
||||
if zipped {
|
||||
// 暂存目录与仓库位于同一文件系统,整个 ZIP 一次 rename 发布。
|
||||
err = os.Rename(stage, filepath.Join(parent, folder))
|
||||
} else {
|
||||
var published []string
|
||||
for _, f := range top {
|
||||
name := f.Name()
|
||||
if err = os.Rename(filepath.Join(stage, name), filepath.Join(parent, name)); err != nil {
|
||||
for _, prev := range published {
|
||||
if rollback := os.Rename(filepath.Join(parent, prev), filepath.Join(stage, prev)); rollback != nil {
|
||||
err = errors.Join(err, fmt.Errorf("回滚导入失败:%w", rollback))
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
published = append(published, name)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.Paths = targets
|
||||
return result, nil
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
package vault
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"html"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var importRef = regexp.MustCompile(`^ {0,3}\[[^\]\r\n]+\]:[ \t]*`)
|
||||
var importSrc = regexp.MustCompile(`(?i)<(?:img|source|video|audio)\b[^>]*?\bsrc\s*=\s*["']([^"']*)["']`)
|
||||
var markdownEscape = regexp.MustCompile("\\\\([!\"#$%&'()*+,\\-./:;<=>?@\\[\\]\\\\^_`{|}~])")
|
||||
|
||||
type importLink struct {
|
||||
start, end int
|
||||
html bool
|
||||
}
|
||||
|
||||
// linkDestination 返回目的地址本身的区间,不含尖括号、标题或闭合括号。
|
||||
func linkDestination(text []byte, start int) (int, int) {
|
||||
for start < len(text) && (text[start] == ' ' || text[start] == '\t') {
|
||||
start++
|
||||
}
|
||||
if start == len(text) {
|
||||
return start, start
|
||||
}
|
||||
if text[start] == '<' {
|
||||
end := start + 1
|
||||
for end < len(text) && text[end] != '>' && text[end] != '\n' && text[end] != '\r' {
|
||||
if text[end] == '\\' && end+1 < len(text) {
|
||||
end++
|
||||
}
|
||||
end++
|
||||
}
|
||||
if end < len(text) && text[end] == '>' {
|
||||
return start + 1, end
|
||||
}
|
||||
return start, start
|
||||
}
|
||||
depth := 0
|
||||
end := start
|
||||
for end < len(text) {
|
||||
c := text[end]
|
||||
if c == '\\' && end+1 < len(text) {
|
||||
end += 2
|
||||
continue
|
||||
}
|
||||
if c == '(' {
|
||||
depth++
|
||||
}
|
||||
if c == ')' {
|
||||
if depth == 0 {
|
||||
break
|
||||
}
|
||||
depth--
|
||||
}
|
||||
if c == ' ' || c == '\t' || c == '\n' || c == '\r' {
|
||||
break
|
||||
}
|
||||
end++
|
||||
}
|
||||
if depth != 0 {
|
||||
return start, start
|
||||
}
|
||||
return start, end
|
||||
}
|
||||
|
||||
func importLinkSpans(text []byte) []importLink {
|
||||
var spans []importLink
|
||||
if m := importRef.FindIndex(text); m != nil {
|
||||
start, end := linkDestination(text, m[1])
|
||||
if end > start {
|
||||
spans = append(spans, importLink{start, end, false})
|
||||
}
|
||||
}
|
||||
for pos := 0; pos < len(text); {
|
||||
i := bytes.Index(text[pos:], []byte("]("))
|
||||
if i < 0 {
|
||||
break
|
||||
}
|
||||
i += pos
|
||||
// 转义的闭括号不是链接语法。
|
||||
escapes := 0
|
||||
for j := i - 1; j >= 0 && text[j] == '\\'; j-- {
|
||||
escapes++
|
||||
}
|
||||
start, end := linkDestination(text, i+2)
|
||||
if escapes%2 == 0 && end > start {
|
||||
spans = append(spans, importLink{start, end, false})
|
||||
}
|
||||
pos = max(i+2, end)
|
||||
}
|
||||
for _, m := range importSrc.FindAllSubmatchIndex(text, -1) {
|
||||
spans = append(spans, importLink{m[2], m[3], true})
|
||||
}
|
||||
return spans
|
||||
}
|
||||
|
||||
func (v *Vault) importLinks(ctx context.Context, note string, content []byte, total *int64) ([]byte, []string, error) {
|
||||
var failure error
|
||||
missing := map[string]bool{}
|
||||
replacements := map[string]string{}
|
||||
transform := func(text []byte) []byte {
|
||||
spans := importLinkSpans(text)
|
||||
// 区间可能来自 HTML 或 Markdown,按原始顺序替换,保持其他字节不变。
|
||||
sort.Slice(spans, func(i, j int) bool { return spans[i].start < spans[j].start })
|
||||
var out []byte
|
||||
last := 0
|
||||
for _, span := range spans {
|
||||
if span.start < last || failure != nil {
|
||||
continue
|
||||
}
|
||||
raw := string(text[span.start:span.end])
|
||||
target := raw
|
||||
if span.html {
|
||||
target = html.UnescapeString(target)
|
||||
} else {
|
||||
target = markdownEscape.ReplaceAllString(target, "$1")
|
||||
}
|
||||
u, err := url.Parse(target)
|
||||
if err != nil || u.IsAbs() || u.Host != "" || strings.HasPrefix(target, "/") || u.Path == "" {
|
||||
continue
|
||||
}
|
||||
rel := path.Clean(path.Join(path.Dir(note), u.Path))
|
||||
if _, err := CleanPath(rel); err != nil {
|
||||
missing[rel] = true
|
||||
continue
|
||||
}
|
||||
info, err := os.Stat(filepath.Join(v.root, filepath.FromSlash(rel)))
|
||||
if os.IsNotExist(err) {
|
||||
missing[rel] = true
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
failure = err
|
||||
continue
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
continue
|
||||
}
|
||||
switch strings.ToLower(path.Ext(rel)) {
|
||||
case ".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".avif", ".apng", ".bmp", ".ico", ".tif", ".tiff":
|
||||
default:
|
||||
continue
|
||||
}
|
||||
replacement, found := replacements[rel]
|
||||
if !found {
|
||||
if err := ctx.Err(); err != nil {
|
||||
failure = err
|
||||
continue
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(v.root, filepath.FromSlash(rel)))
|
||||
if err != nil {
|
||||
failure = err
|
||||
continue
|
||||
}
|
||||
// 只统计新生成的副本;已有同名附件必须与哈希对应的内容一致。
|
||||
dest := filepath.Join(v.root, filepath.FromSlash(AssetsDir(note)), assetName(data, path.Base(rel)))
|
||||
existing, readErr := os.ReadFile(dest)
|
||||
if readErr != nil && !os.IsNotExist(readErr) {
|
||||
failure = readErr
|
||||
continue
|
||||
}
|
||||
if readErr == nil && !bytes.Equal(data, existing) {
|
||||
failure = fmt.Errorf("%w:附件名称与内容冲突", ErrInvalidImport)
|
||||
continue
|
||||
}
|
||||
if os.IsNotExist(readErr) {
|
||||
if int64(len(data)) > maxImportExpanded-*total {
|
||||
failure = ErrTooLarge
|
||||
continue
|
||||
}
|
||||
*total += int64(len(data))
|
||||
}
|
||||
replacement, err = v.SaveAsset(note, path.Base(rel), bytes.NewReader(data), importAssetLimit)
|
||||
if err != nil {
|
||||
failure = err
|
||||
continue
|
||||
}
|
||||
replacements[rel] = replacement
|
||||
}
|
||||
if u.RawQuery != "" {
|
||||
replacement += "?" + u.RawQuery
|
||||
}
|
||||
if u.Fragment != "" {
|
||||
replacement += "#" + u.EscapedFragment()
|
||||
}
|
||||
if span.html {
|
||||
replacement = html.EscapeString(replacement)
|
||||
}
|
||||
out = append(out, text[last:span.start]...)
|
||||
out = append(out, replacement...)
|
||||
last = span.end
|
||||
}
|
||||
return append(out, text[last:]...)
|
||||
}
|
||||
updated := transformMarkdownLines(content, importRef, func(line []byte) []byte { return transformOutsideCode(line, transform) })
|
||||
warnings := []string{}
|
||||
if len(missing) > 0 {
|
||||
warnings = append(warnings, note+":"+strconv.Itoa(len(missing))+" 个本地链接的目标不在导入内容中,已保留原链接")
|
||||
}
|
||||
return updated, warnings, failure
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
package vault
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type zipEntry struct {
|
||||
name, content string
|
||||
mode fs.FileMode
|
||||
}
|
||||
|
||||
func importZip(t *testing.T, entries ...zipEntry) ImportFile {
|
||||
t.Helper()
|
||||
var b bytes.Buffer
|
||||
w := zip.NewWriter(&b)
|
||||
for _, e := range entries {
|
||||
h := &zip.FileHeader{Name: e.name, Method: zip.Deflate}
|
||||
if e.mode != 0 {
|
||||
h.SetMode(e.mode)
|
||||
}
|
||||
f, err := w.CreateHeader(h)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := f.Write([]byte(e.content)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data := b.Bytes()
|
||||
return ImportFile{Name: "知识库.md.zip", Reader: bytes.NewReader(data), Size: int64(len(data))}
|
||||
}
|
||||
|
||||
func markdownFile(name, content string) ImportFile {
|
||||
return ImportFile{Name: name, Reader: strings.NewReader(content), Size: int64(len(content))}
|
||||
}
|
||||
|
||||
func TestImportZipAssetsAndMove(t *testing.T) {
|
||||
v := newVault(t)
|
||||
content := "# 标题\n\n.png> \"说明\")\n\n![引用][图]\n\n[图]: assets/图%20(1).png\n\n<img src=\"assets/图%20(1).png\">\n\n[子笔记](子目录/子笔记.md)\n\n\n\n\n\n`.png)`\n\n```md\n.png)\n```\n\n> ```\n> .png)\n> ```\n\n .png)\n"
|
||||
source := importZip(t, zipEntry{name: "目录/笔记.md", content: content}, zipEntry{name: "目录/子目录/子笔记.md", content: "# 子笔记"}, zipEntry{name: "目录/assets/图 (1).png", content: "image"}, zipEntry{name: "未引用.txt", content: "附件"})
|
||||
result, err := v.Import(context.Background(), "", "知识库", []ImportFile{source})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Notes != 2 || result.Assets != 3 || len(result.Warnings) != 1 || len(result.Paths) != 1 || result.Paths[0] != "知识库" {
|
||||
t.Fatalf("结果不符: %+v", result)
|
||||
}
|
||||
body, _, err := v.Read("知识库/目录/笔记.md")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
link := EscapeLinkPath("笔记.assets") + "/" + assetName([]byte("image"), "x.png")
|
||||
expected := strings.Replace(content, "<assets/图 (1).png>", "<"+link+">", 1)
|
||||
expected = strings.Replace(expected, "[图]: assets/图%20(1).png", "[图]: "+link, 1)
|
||||
expected = strings.Replace(expected, "src=\"assets/图%20(1).png\"", "src=\""+link+"\"", 1)
|
||||
if string(body) != expected {
|
||||
t.Fatalf("正文变更不符:\n%s", body)
|
||||
}
|
||||
if err := v.Mkdir("移动目标"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := v.Move("知识库/目录/笔记.md", "移动目标/改名.md"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
moved, _, err := v.Read("移动目标/改名.md")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(moved), "改名.assets/") {
|
||||
t.Fatal("图片链接未跟随改名")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(v.root, "移动目标/改名.assets", assetName([]byte("image"), "x.png"))); err != nil {
|
||||
t.Fatal("附件未跟随", err)
|
||||
}
|
||||
original, err := os.ReadFile(filepath.Join(v.root, "知识库/目录/assets/图 (1).png"))
|
||||
if err != nil || string(original) != "image" {
|
||||
t.Fatal("原始附件丢失", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportRejectsInvalidWithoutPublishing(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
entries []zipEntry
|
||||
want error
|
||||
}{
|
||||
{"越界", []zipEntry{{name: "../outside.md", content: "x"}}, ErrInvalidImport},
|
||||
{"隐藏路径", []zipEntry{{name: ".rawmark/x.md", content: "x"}}, ErrInvalidImport},
|
||||
{"重复路径", []zipEntry{{name: "a.md", content: "x"}, {name: "a.md", content: "y"}}, ErrInvalidImport},
|
||||
{"符号链接", []zipEntry{{name: "a.md", content: "x", mode: os.ModeSymlink | 0o777}}, ErrInvalidImport},
|
||||
{"目录文件冲突", []zipEntry{{name: "a/"}, {name: "a", content: "x"}, {name: "b.md", content: "x"}}, ErrInvalidImport},
|
||||
{"文件目录冲突反向", []zipEntry{{name: "a", content: "x"}, {name: "a/"}, {name: "b.md", content: "x"}}, ErrInvalidImport},
|
||||
{"无笔记", []zipEntry{{name: "a.png", content: "x"}}, ErrInvalidImport},
|
||||
{"非法编码", []zipEntry{{name: "a.md", content: "\xff"}}, ErrInvalidImport},
|
||||
{"附件目录内笔记", []zipEntry{{name: "a.assets/a.md", content: "x"}}, ErrInvalidImport},
|
||||
{"文件目录冲突", []zipEntry{{name: "a", content: "x"}, {name: "a/b.md", content: "y"}}, ErrInvalidImport},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
v := newVault(t)
|
||||
_, err := v.Import(context.Background(), "", "导入", []ImportFile{importZip(t, tc.entries...)})
|
||||
if !errors.Is(err, tc.want) {
|
||||
t.Fatalf("错误不符: %v", err)
|
||||
}
|
||||
entries, err := os.ReadDir(v.root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(entries) != 1 || entries[0].Name() != metaDir {
|
||||
t.Fatal("失败后留下可见内容或临时目录")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportBatchConflictAndCancellation(t *testing.T) {
|
||||
v := newVault(t)
|
||||
if _, err := v.Write("b.md", []byte("原文"), Precondition{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sources := []ImportFile{markdownFile("a.md", "新文"), markdownFile("b.md", "覆盖")}
|
||||
if _, err := v.Import(context.Background(), "", "", sources); !errors.Is(err, ErrExists) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := v.Read("a.md"); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatal("批量冲突产生了半成品", err)
|
||||
}
|
||||
body, _, _ := v.Read("b.md")
|
||||
if string(body) != "原文" {
|
||||
t.Fatal("原文件被覆盖")
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := v.Import(ctx, "", "", sources[:1]); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := v.Import(context.Background(), "不存在", "", sources[:1]); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := v.Import(context.Background(), "", "", []ImportFile{markdownFile("a.md", "\xef\xbb\xbf正文"), markdownFile("c.md", "")})
|
||||
if err != nil || result.Notes != 2 || len(result.Warnings) != 1 {
|
||||
t.Fatalf("结果不符: %+v %v", result, err)
|
||||
}
|
||||
body, _, _ = v.Read("a.md")
|
||||
if string(body) != "正文" {
|
||||
t.Fatal("UTF-8 BOM 未处理")
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportLimitsAndCorruptZip(t *testing.T) {
|
||||
v := newVault(t)
|
||||
source := markdownFile("large.md", strings.Repeat("a", importNoteLimit+1))
|
||||
if _, err := v.Import(context.Background(), "", "", []ImportFile{source}); !errors.Is(err, ErrTooLarge) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
source.Size = MaxImportBytes + 1
|
||||
if _, err := v.Import(context.Background(), "", "", []ImportFile{source}); !errors.Is(err, ErrTooLarge) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
broken := markdownFile("broken.zip", "not zip")
|
||||
if _, err := v.Import(context.Background(), "", "导入", []ImportFile{broken}); !errors.Is(err, ErrInvalidImport) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportCorruptZipContent(t *testing.T) {
|
||||
v := newVault(t)
|
||||
var buffer bytes.Buffer
|
||||
writer := zip.NewWriter(&buffer)
|
||||
f, err := writer.CreateHeader(&zip.FileHeader{Name: "n.md", Method: zip.Store})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
payload := []byte("unique markdown payload")
|
||||
if _, err := f.Write(payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data := buffer.Bytes()
|
||||
offset := bytes.Index(data, payload)
|
||||
if offset < 0 {
|
||||
t.Fatal("找不到测试负载")
|
||||
}
|
||||
data[offset] ^= 1
|
||||
_, err = v.Import(context.Background(), "", "损坏", []ImportFile{{Name: "broken.zip", Reader: bytes.NewReader(data), Size: int64(len(data))}})
|
||||
if !errors.Is(err, ErrInvalidImport) {
|
||||
t.Fatalf("应拒绝 CRC 损坏: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(v.root, "损坏")); !os.IsNotExist(err) {
|
||||
t.Fatal("损坏内容被发布", err)
|
||||
}
|
||||
}
|
||||
+13
-6
@@ -16,6 +16,14 @@ func RewriteAssetLinks(content []byte, oldBase, newBase string) []byte {
|
||||
refDef := regexp.MustCompile(`^( {0,3}\[[^\]]+\]:\s*<?)` + target)
|
||||
repl := []byte("${1}${2}" + strings.ReplaceAll(EscapeLinkPath(newBase+AssetsExt), "$", "$$") + "/")
|
||||
|
||||
return transformMarkdownLines(content, refDef, func(line []byte) []byte {
|
||||
line = refDef.ReplaceAll(line, repl)
|
||||
return transformOutsideCode(line, func(text []byte) []byte { return inline.ReplaceAll(text, repl) })
|
||||
})
|
||||
}
|
||||
|
||||
// transformMarkdownLines 复用链接改写的容器与代码块识别,避免导入时修改代码示例。
|
||||
func transformMarkdownLines(content []byte, refDef *regexp.Regexp, transform func([]byte) []byte) []byte {
|
||||
lines := bytes.SplitAfter(content, []byte("\n"))
|
||||
var fence []byte // 当前所在围栏代码块的围栏字符串,nil 表示不在代码块中
|
||||
var containers []linkContainer
|
||||
@@ -111,8 +119,7 @@ func RewriteAssetLinks(content []byte, oldBase, newBase string) []byte {
|
||||
continue
|
||||
}
|
||||
paragraph = !paragraphBoundary.Match(trimmed) && !refDef.Match(trimmed)
|
||||
line = refDef.ReplaceAll(line, repl)
|
||||
lines[i] = replaceOutsideCode(line, inline, repl)
|
||||
lines[i] = transform(line)
|
||||
}
|
||||
return bytes.Join(lines, nil)
|
||||
}
|
||||
@@ -161,8 +168,8 @@ func fenceOpen(line []byte) []byte {
|
||||
return rest[:n]
|
||||
}
|
||||
|
||||
// replaceOutsideCode 只在行内代码 `…` 之外的片段上做替换
|
||||
func replaceOutsideCode(line []byte, re *regexp.Regexp, repl []byte) []byte {
|
||||
// transformOutsideCode 只在行内代码 `…` 之外的片段上做转换。
|
||||
func transformOutsideCode(line []byte, transform func([]byte) []byte) []byte {
|
||||
var out []byte
|
||||
for len(line) > 0 {
|
||||
start := bytes.IndexByte(line, '`')
|
||||
@@ -178,9 +185,9 @@ func replaceOutsideCode(line []byte, re *regexp.Regexp, repl []byte) []byte {
|
||||
break
|
||||
}
|
||||
codeEnd := start + n + end + n
|
||||
out = append(out, re.ReplaceAll(line[:start], repl)...)
|
||||
out = append(out, transform(line[:start])...)
|
||||
out = append(out, line[start:codeEnd]...)
|
||||
line = line[codeEnd:]
|
||||
}
|
||||
return append(out, re.ReplaceAll(line, repl)...)
|
||||
return append(out, transform(line)...)
|
||||
}
|
||||
+10
-6
@@ -696,12 +696,7 @@ func (v *Vault) SaveAsset(note, origName string, r io.Reader, limit int64) (stri
|
||||
if int64(len(data)) > limit {
|
||||
return "", ErrTooLarge
|
||||
}
|
||||
ext := strings.ToLower(filepath.Ext(origName))
|
||||
if !extPattern.MatchString(ext) {
|
||||
ext = sniffExt(data)
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
name := hex.EncodeToString(sum[:8]) + ext
|
||||
name := assetName(data, origName)
|
||||
dir := AssetsDir(p)
|
||||
v.mu.Lock()
|
||||
defer v.mu.Unlock()
|
||||
@@ -722,6 +717,15 @@ func (v *Vault) SaveAsset(note, origName string, r io.Reader, limit int64) (stri
|
||||
return EscapeLinkPath(path.Base(AssetsDir(note))) + "/" + name, nil
|
||||
}
|
||||
|
||||
func assetName(data []byte, origName string) string {
|
||||
ext := strings.ToLower(filepath.Ext(origName))
|
||||
if !extPattern.MatchString(ext) {
|
||||
ext = sniffExt(data)
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
return hex.EncodeToString(sum[:8]) + ext
|
||||
}
|
||||
|
||||
func sniffExt(data []byte) string {
|
||||
switch http.DetectContentType(data) {
|
||||
case "image/png":
|
||||
|
||||
Reference in new issue
Block a user