Add Markdown and ZIP notebook import

This commit is contained in:
eezd committed 2026-10-21 12:56:38 +08:00
1 parent 903a617439
commit 34d2acbbd5
18 files changed
+1166 -16

No files matched your search

+2 -1
View File
@@ -84,6 +84,7 @@ func (s *Server) routes() []route {
{"DELETE /api/v1/files/{path...}", authed, s.deleteFile},
{"POST /api/v1/move", authed, s.move},
{"POST /api/v1/folders", authed, s.mkdir},
{"POST /api/v1/import", authed, s.importNotes},
{"POST /api/v1/assets", authed, s.uploadAsset},
{"POST /api/v1/assets/import", authed, s.importAsset},
{"GET /api/v1/raw/{path...}", authed, s.raw},
@@ -492,7 +493,7 @@ func vaultError(w http.ResponseWriter, r *http.Request, err error) {
switch {
case errors.As(err, &conflict):
writeJSON(w, 412, map[string]string{"error": err.Error(), "rev": conflict.Rev})
case errors.Is(err, vault.ErrInvalidPath):
case errors.Is(err, vault.ErrInvalidPath), errors.Is(err, vault.ErrInvalidImport):
writeError(w, 400, err.Error())
case errors.Is(err, vault.ErrNotFound):
writeError(w, 404, err.Error())
+44
View File
@@ -0,0 +1,44 @@
package api
import (
"errors"
"net/http"
"rawmark/internal/vault"
)
func (s *Server) importNotes(w http.ResponseWriter, r *http.Request) {
// 总请求上限包含表单开销,文件总量另由 vault 校验。
r.Body = http.MaxBytesReader(w, r.Body, vault.MaxImportBytes+(1<<20))
if err := r.ParseMultipartForm(8 << 20); err != nil {
var limit *http.MaxBytesError
if errors.As(err, &limit) {
writeError(w, 413, "导入文件总量不能超过 100 MB")
} else {
writeError(w, 400, "无法读取导入表单")
}
return
}
defer r.MultipartForm.RemoveAll()
headers := r.MultipartForm.File["files"]
if len(headers) == 0 || len(headers) > 10_000 {
writeError(w, 400, "请选择 Markdown 文件或一个 ZIP")
return
}
files := make([]vault.ImportFile, 0, len(headers))
for _, h := range headers {
f, err := h.Open()
if err != nil {
internalError(w, r, err)
return
}
defer f.Close()
files = append(files, vault.ImportFile{Name: h.Filename, Reader: f, Size: h.Size})
}
result, err := s.vault.Import(r.Context(), r.FormValue("dir"), r.FormValue("folder"), files)
if err != nil {
vaultError(w, r, err)
return
}
writeJSON(w, http.StatusCreated, result)
}
+117
View File
@@ -0,0 +1,117 @@
package api
import (
"archive/zip"
"bytes"
"encoding/json"
"io"
"mime/multipart"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"rawmark/internal/vault"
)
func TestImportNotesAPI(t *testing.T) {
f := newFixture(t)
send := func(client *http.Client, token, dir, folder, name string, data []byte) *http.Response {
t.Helper()
var body bytes.Buffer
form := multipart.NewWriter(&body)
if err := form.WriteField("dir", dir); err != nil {
t.Fatal(err)
}
if err := form.WriteField("folder", folder); err != nil {
t.Fatal(err)
}
file, err := form.CreateFormFile("files", name)
if err != nil {
t.Fatal(err)
}
if _, err := file.Write(data); err != nil {
t.Fatal(err)
}
if err := form.Close(); err != nil {
t.Fatal(err)
}
req, err := http.NewRequest("POST", f.url+"/api/v1/import", &body)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Content-Type", form.FormDataContentType())
if token != "" {
req.Header.Set("Authorization", "Bearer "+token)
}
res, err := client.Do(req)
if err != nil {
t.Fatal(err)
}
return res
}
readToken := decode[struct{ Token string }](t, f.do(f.admin, "POST", "/api/v1/tokens", `{"name":"只读","scope":"read"}`, nil)).Token
for _, tc := range []struct {
client *http.Client
token string
status int
}{{newClient(), "", 401}, {newClient(), readToken, 403}} {
res := send(tc.client, tc.token, "", "", "n.md", []byte("正文"))
res.Body.Close()
if res.StatusCode != tc.status {
t.Fatalf("权限状态不符: %d", res.StatusCode)
}
}
res := send(f.admin, "", "", "", "n.md", []byte("正文"))
if res.StatusCode != 201 {
t.Fatalf("导入失败: %d", res.StatusCode)
}
var result vault.ImportResult
if err := json.NewDecoder(res.Body).Decode(&result); err != nil {
t.Fatal(err)
}
res.Body.Close()
if result.Notes != 1 || len(result.Paths) != 1 || result.Paths[0] != "n.md" || result.Warnings == nil {
t.Fatalf("响应不符: %+v", result)
}
res = send(f.admin, "", "", "", "n.md", []byte("覆盖"))
res.Body.Close()
if res.StatusCode != 409 {
t.Fatalf("应拒绝冲突: %d", res.StatusCode)
}
data, err := os.ReadFile(filepath.Join(f.root, "n.md"))
if err != nil || string(data) != "正文" {
t.Fatal("原文被覆盖", err)
}
writeToken := decode[struct{ Token string }](t, f.do(f.admin, "POST", "/api/v1/tokens", `{"name":"写入","scope":"write"}`, nil)).Token
var b bytes.Buffer
zw := zip.NewWriter(&b)
zf, err := zw.Create("子目录/笔记.md")
if err != nil {
t.Fatal(err)
}
if _, err := io.WriteString(zf, "# ZIP"); err != nil {
t.Fatal(err)
}
if err := zw.Close(); err != nil {
t.Fatal(err)
}
res = send(newClient(), writeToken, "", "知识库", "export.md.zip", b.Bytes())
res.Body.Close()
if res.StatusCode != 201 {
t.Fatalf("写入令牌导入失败: %d", res.StatusCode)
}
for _, tc := range []struct {
dir, folder, name, data string
status int
}{
{"", "", "x.zip", "invalid", 400}, {"不存在", "", "x.md", "x", 404}, {"n.md", "", "x.md", "x", 400}, {"", "", "x.md", strings.Repeat("a", (20<<20)+1), 413},
} {
res := send(f.admin, "", tc.dir, tc.folder, tc.name, []byte(tc.data))
res.Body.Close()
if res.StatusCode != tc.status {
t.Fatalf("%s 状态不符: %d", tc.name, res.StatusCode)
}
}
}
+96
View File
@@ -2180,6 +2180,102 @@
}
]
}
},
"/import": {
"post": {
"summary": "导入 Markdown 文件或 ZIP 知识库",
"tags": [
"文件"
],
"description": "多选 .md 或单个 ZIP,总上传不超过 100 MB;ZIP 必须指定新建文件夹名称。目标目录必须存在,冲突不覆盖。解压及整理后上限 512 MiB、10,000 项,单篇笔记 20 MiB、附件 50 MiB。Markdown 与 ZIP 文件名须为 UTF-8。ZIP 层级保留,本地图片生成笔记独立附件,原附件保留;外部链接不下载,缺失本地目标以警告返回。",
"requestBody": {
"required": true,
"content": {
"multipart/form-data": {
"schema": {
"type": "object",
"required": [
"files"
],
"properties": {
"dir": {
"type": "string",
"description": "目标目录,空字符串为根目录"
},
"folder": {
"type": "string",
"description": "ZIP 的新建文件夹名称;普通 Markdown 留空"
},
"files": {
"type": "array",
"items": {
"type": "string",
"format": "binary"
}
}
}
}
}
}
},
"responses": {
"201": {
"description": "导入成功",
"content": {
"application/json": {
"schema": {
"type": "object",
"required": [
"paths",
"notes",
"assets",
"warnings"
],
"properties": {
"paths": {
"type": "array",
"items": {
"type": "string"
}
},
"notes": {
"type": "integer"
},
"assets": {
"type": "integer",
"description": "附件文件总数,包含原始附件和独立副本"
},
"warnings": {
"type": "array",
"items": {
"type": "string"
}
}
}
}
}
}
},
"400": {
"description": "格式、路径或编码无效"
},
"401": {
"description": "未登录或令牌无效"
},
"403": {
"description": "只读令牌不能导入"
},
"404": {
"description": "目标目录不存在"
},
"409": {
"description": "目标已存在,不覆盖"
},
"413": {
"description": "超过上传、解压、文件数量或单文件大小限制"
}
}
}
}
},
"components": {
+289
View File
@@ -0,0 +1,289 @@
package vault
import (
"archive/zip"
"bytes"
"compress/flate"
"context"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path"
"path/filepath"
"sort"
"strings"
"syscall"
"unicode/utf8"
)
const (
MaxImportBytes = 100_000_000
maxImportExpanded = 512 << 20
maxImportEntries = 10_000
importNoteLimit = 20 << 20
importAssetLimit = 50 << 20
)
var ErrInvalidImport = errors.New("导入内容无效")
// ImportFile 的读取范围是单个上传文件,ZIP 与普通 Markdown 共用这一输入边界。
type ImportFile struct {
Name string
Reader io.ReaderAt
Size int64
}
type ImportResult struct {
Paths []string `json:"paths"`
Notes int `json:"notes"`
Assets int `json:"assets"`
Warnings []string `json:"warnings"`
}
// Import 在隐藏目录内整理完整内容,提交前统一检查冲突,绝不覆盖已有文件。
func (v *Vault) Import(ctx context.Context, dir, folder string, files []ImportFile) (result ImportResult, err error) {
result = ImportResult{Paths: []string{}, Warnings: []string{}}
if len(files) == 0 || len(files) > maxImportEntries {
return result, fmt.Errorf("%w:请选择 Markdown 文件或一个 ZIP", ErrInvalidImport)
}
zipped := len(files) == 1 && strings.EqualFold(path.Ext(files[0].Name), ".zip")
if zipped {
if _, err := CleanPath(folder); err != nil || strings.Contains(folder, "/") || strings.HasSuffix(folder, AssetsExt) {
return result, fmt.Errorf("%w:导入文件夹名称无效", ErrInvalidImport)
}
} else if folder != "" {
return result, fmt.Errorf("%w:普通 Markdown 不使用外层文件夹", ErrInvalidImport)
}
if dir != "" {
if _, err := CleanPath(dir); err != nil {
return result, err
}
}
var uploadBytes int64
for _, f := range files {
if f.Size < 0 || f.Size > MaxImportBytes-uploadBytes {
return result, ErrTooLarge
}
uploadBytes += f.Size
}
stage, err := os.MkdirTemp(v.root, tmpPrefix+"import-")
if err != nil {
return result, err
}
defer func() {
if cleanup := os.RemoveAll(stage); cleanup != nil {
err = errors.Join(err, fmt.Errorf("清理导入临时目录失败:%w", cleanup))
}
}()
staged := &Vault{root: stage}
entries := map[string]struct{}{}
var total int64
var notes []string
// 每个来源文件只写一次;目录与文件重名也会在暂存阶段报错。
write := func(name string, isDir bool, reader io.Reader) error {
if err := ctx.Err(); err != nil {
return err
}
if _, err := CleanPath(name); err != nil || !utf8.ValidString(name) {
return fmt.Errorf("%w:ZIP 含有非法路径", ErrInvalidImport)
}
if _, exists := entries[name]; exists {
return fmt.Errorf("%w:存在重复路径", ErrInvalidImport)
}
entries[name] = struct{}{}
dest := filepath.Join(stage, filepath.FromSlash(name))
if isDir {
err := os.MkdirAll(dest, 0o755)
if errors.Is(err, fs.ErrExist) || errors.Is(err, syscall.ENOTDIR) {
return fmt.Errorf("%w:文件与目录路径冲突", ErrInvalidImport)
}
return err
}
if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
if errors.Is(err, syscall.ENOTDIR) || errors.Is(err, fs.ErrExist) {
return fmt.Errorf("%w:文件与目录路径冲突", ErrInvalidImport)
}
return err
}
f, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
if err != nil {
if errors.Is(err, fs.ErrExist) || errors.Is(err, syscall.ENOTDIR) {
return fmt.Errorf("%w:文件与目录路径冲突", ErrInvalidImport)
}
return err
}
limit := int64(importAssetLimit)
if IsNote(name) {
limit = importNoteLimit
notes = append(notes, name)
}
limit = min(limit, maxImportExpanded-total)
n, copyErr := io.Copy(f, io.LimitReader(reader, limit+1))
closeErr := f.Close()
total += n
if n > limit {
return ErrTooLarge
}
return errors.Join(copyErr, closeErr)
}
if zipped {
z, err := zip.NewReader(files[0].Reader, files[0].Size)
if err != nil {
return result, fmt.Errorf("%w:无法读取 ZIP:%w", ErrInvalidImport, err)
}
if len(z.File) > maxImportEntries {
return result, ErrTooLarge
}
for _, f := range z.File {
if !f.Mode().IsRegular() && !f.Mode().IsDir() {
return result, fmt.Errorf("%w:ZIP 不允许符号链接或特殊文件", ErrInvalidImport)
}
name := strings.TrimSuffix(f.Name, "/")
r, err := f.Open()
if err != nil {
return result, fmt.Errorf("%w:无法读取 ZIP 条目:%w", ErrInvalidImport, err)
}
err = write(name, f.FileInfo().IsDir(), r)
err = errors.Join(err, r.Close())
if err != nil {
var corrupt flate.CorruptInputError
if errors.Is(err, zip.ErrChecksum) || errors.Is(err, io.ErrUnexpectedEOF) || errors.As(err, &corrupt) {
err = fmt.Errorf("%w:ZIP 内容损坏:%w", ErrInvalidImport, err)
}
return result, err
}
}
} else {
for _, f := range files {
if !IsNote(f.Name) || strings.Contains(f.Name, "/") {
return result, fmt.Errorf("%w:只接受 .md 文件或单个 ZIP", ErrInvalidImport)
}
if err := write(f.Name, false, io.NewSectionReader(f.Reader, 0, f.Size)); err != nil {
return result, err
}
}
}
if len(notes) == 0 {
return result, fmt.Errorf("%w:没有可导入的 Markdown 笔记", ErrInvalidImport)
}
sort.Strings(notes)
for _, note := range notes {
if err := ctx.Err(); err != nil {
return result, err
}
for _, seg := range strings.Split(path.Dir(note), "/") {
if strings.HasSuffix(seg, AssetsExt) {
return result, fmt.Errorf("%w:笔记不能放在 .assets 附件目录内", ErrInvalidImport)
}
}
content, _, err := staged.Read(note)
if err != nil {
return result, err
}
if !utf8.Valid(content) || bytes.IndexByte(content, 0) >= 0 {
return result, fmt.Errorf("%w:Markdown 必须是 UTF-8 文本", ErrInvalidImport)
}
content = bytes.TrimPrefix(content, []byte{0xef, 0xbb, 0xbf})
updated, warnings, err := staged.importLinks(ctx, note, content, &total)
if err != nil {
return result, err
}
result.Warnings = append(result.Warnings, warnings...)
if len(updated) > importNoteLimit || total+int64(len(updated)-len(content)) > maxImportExpanded {
return result, ErrTooLarge
}
total += int64(len(updated) - len(content))
if err := os.WriteFile(filepath.Join(stage, filepath.FromSlash(note)), updated, 0o644); err != nil {
return result, err
}
}
result.Notes = len(notes)
// 附件数量包含原始附件及为各笔记生成的独立副本。
count := 0
err = filepath.WalkDir(stage, func(p string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
count++
if count > maxImportEntries+1 {
return ErrTooLarge
}
if !d.IsDir() && !IsNote(d.Name()) {
result.Assets++
}
return nil
})
if err != nil {
return result, err
}
top, err := os.ReadDir(stage)
if err != nil {
return result, err
}
targets := []string{}
if zipped {
targets = append(targets, path.Join(dir, folder))
} else {
for _, f := range top {
targets = append(targets, path.Join(dir, f.Name()))
}
}
v.mu.Lock()
defer v.mu.Unlock()
parent := v.root
if dir != "" {
parent, err = v.abs(dir)
if err != nil {
return result, err
}
}
info, err := os.Stat(parent)
if errors.Is(err, fs.ErrNotExist) {
return result, ErrNotFound
}
if err != nil {
return result, err
}
if !info.IsDir() {
return result, fmt.Errorf("%w:目标不是文件夹", ErrInvalidImport)
}
for _, target := range targets {
dest, err := v.abs(target)
if err != nil {
return result, err
}
if _, err := os.Lstat(dest); err == nil {
return result, fmt.Errorf("%w:%s,请修改名称", ErrExists, target)
} else if !errors.Is(err, fs.ErrNotExist) {
return result, err
}
}
if err := ctx.Err(); err != nil {
return result, err
}
if zipped {
// 暂存目录与仓库位于同一文件系统,整个 ZIP 一次 rename 发布。
err = os.Rename(stage, filepath.Join(parent, folder))
} else {
var published []string
for _, f := range top {
name := f.Name()
if err = os.Rename(filepath.Join(stage, name), filepath.Join(parent, name)); err != nil {
for _, prev := range published {
if rollback := os.Rename(filepath.Join(parent, prev), filepath.Join(stage, prev)); rollback != nil {
err = errors.Join(err, fmt.Errorf("回滚导入失败:%w", rollback))
}
}
break
}
published = append(published, name)
}
}
if err != nil {
return result, err
}
result.Paths = targets
return result, nil
}
+211
View File
@@ -0,0 +1,211 @@
package vault
import (
"bytes"
"context"
"fmt"
"html"
"net/url"
"os"
"path"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
)
var importRef = regexp.MustCompile(`^ {0,3}\[[^\]\r\n]+\]:[ \t]*`)
var importSrc = regexp.MustCompile(`(?i)<(?:img|source|video|audio)\b[^>]*?\bsrc\s*=\s*["']([^"']*)["']`)
var markdownEscape = regexp.MustCompile("\\\\([!\"#$%&'()*+,\\-./:;<=>?@\\[\\]\\\\^_`{|}~])")
type importLink struct {
start, end int
html bool
}
// linkDestination 返回目的地址本身的区间,不含尖括号、标题或闭合括号。
func linkDestination(text []byte, start int) (int, int) {
for start < len(text) && (text[start] == ' ' || text[start] == '\t') {
start++
}
if start == len(text) {
return start, start
}
if text[start] == '<' {
end := start + 1
for end < len(text) && text[end] != '>' && text[end] != '\n' && text[end] != '\r' {
if text[end] == '\\' && end+1 < len(text) {
end++
}
end++
}
if end < len(text) && text[end] == '>' {
return start + 1, end
}
return start, start
}
depth := 0
end := start
for end < len(text) {
c := text[end]
if c == '\\' && end+1 < len(text) {
end += 2
continue
}
if c == '(' {
depth++
}
if c == ')' {
if depth == 0 {
break
}
depth--
}
if c == ' ' || c == '\t' || c == '\n' || c == '\r' {
break
}
end++
}
if depth != 0 {
return start, start
}
return start, end
}
func importLinkSpans(text []byte) []importLink {
var spans []importLink
if m := importRef.FindIndex(text); m != nil {
start, end := linkDestination(text, m[1])
if end > start {
spans = append(spans, importLink{start, end, false})
}
}
for pos := 0; pos < len(text); {
i := bytes.Index(text[pos:], []byte("]("))
if i < 0 {
break
}
i += pos
// 转义的闭括号不是链接语法。
escapes := 0
for j := i - 1; j >= 0 && text[j] == '\\'; j-- {
escapes++
}
start, end := linkDestination(text, i+2)
if escapes%2 == 0 && end > start {
spans = append(spans, importLink{start, end, false})
}
pos = max(i+2, end)
}
for _, m := range importSrc.FindAllSubmatchIndex(text, -1) {
spans = append(spans, importLink{m[2], m[3], true})
}
return spans
}
func (v *Vault) importLinks(ctx context.Context, note string, content []byte, total *int64) ([]byte, []string, error) {
var failure error
missing := map[string]bool{}
replacements := map[string]string{}
transform := func(text []byte) []byte {
spans := importLinkSpans(text)
// 区间可能来自 HTML 或 Markdown,按原始顺序替换,保持其他字节不变。
sort.Slice(spans, func(i, j int) bool { return spans[i].start < spans[j].start })
var out []byte
last := 0
for _, span := range spans {
if span.start < last || failure != nil {
continue
}
raw := string(text[span.start:span.end])
target := raw
if span.html {
target = html.UnescapeString(target)
} else {
target = markdownEscape.ReplaceAllString(target, "$1")
}
u, err := url.Parse(target)
if err != nil || u.IsAbs() || u.Host != "" || strings.HasPrefix(target, "/") || u.Path == "" {
continue
}
rel := path.Clean(path.Join(path.Dir(note), u.Path))
if _, err := CleanPath(rel); err != nil {
missing[rel] = true
continue
}
info, err := os.Stat(filepath.Join(v.root, filepath.FromSlash(rel)))
if os.IsNotExist(err) {
missing[rel] = true
continue
}
if err != nil {
failure = err
continue
}
if !info.Mode().IsRegular() {
continue
}
switch strings.ToLower(path.Ext(rel)) {
case ".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".avif", ".apng", ".bmp", ".ico", ".tif", ".tiff":
default:
continue
}
replacement, found := replacements[rel]
if !found {
if err := ctx.Err(); err != nil {
failure = err
continue
}
data, err := os.ReadFile(filepath.Join(v.root, filepath.FromSlash(rel)))
if err != nil {
failure = err
continue
}
// 只统计新生成的副本;已有同名附件必须与哈希对应的内容一致。
dest := filepath.Join(v.root, filepath.FromSlash(AssetsDir(note)), assetName(data, path.Base(rel)))
existing, readErr := os.ReadFile(dest)
if readErr != nil && !os.IsNotExist(readErr) {
failure = readErr
continue
}
if readErr == nil && !bytes.Equal(data, existing) {
failure = fmt.Errorf("%w:附件名称与内容冲突", ErrInvalidImport)
continue
}
if os.IsNotExist(readErr) {
if int64(len(data)) > maxImportExpanded-*total {
failure = ErrTooLarge
continue
}
*total += int64(len(data))
}
replacement, err = v.SaveAsset(note, path.Base(rel), bytes.NewReader(data), importAssetLimit)
if err != nil {
failure = err
continue
}
replacements[rel] = replacement
}
if u.RawQuery != "" {
replacement += "?" + u.RawQuery
}
if u.Fragment != "" {
replacement += "#" + u.EscapedFragment()
}
if span.html {
replacement = html.EscapeString(replacement)
}
out = append(out, text[last:span.start]...)
out = append(out, replacement...)
last = span.end
}
return append(out, text[last:]...)
}
updated := transformMarkdownLines(content, importRef, func(line []byte) []byte { return transformOutsideCode(line, transform) })
warnings := []string{}
if len(missing) > 0 {
warnings = append(warnings, note+":"+strconv.Itoa(len(missing))+" 个本地链接的目标不在导入内容中,已保留原链接")
}
return updated, warnings, failure
}
+204
View File
@@ -0,0 +1,204 @@
package vault
import (
"archive/zip"
"bytes"
"context"
"errors"
"io/fs"
"os"
"path/filepath"
"strings"
"testing"
)
type zipEntry struct {
name, content string
mode fs.FileMode
}
func importZip(t *testing.T, entries ...zipEntry) ImportFile {
t.Helper()
var b bytes.Buffer
w := zip.NewWriter(&b)
for _, e := range entries {
h := &zip.FileHeader{Name: e.name, Method: zip.Deflate}
if e.mode != 0 {
h.SetMode(e.mode)
}
f, err := w.CreateHeader(h)
if err != nil {
t.Fatal(err)
}
if _, err := f.Write([]byte(e.content)); err != nil {
t.Fatal(err)
}
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
data := b.Bytes()
return ImportFile{Name: "知识库.md.zip", Reader: bytes.NewReader(data), Size: int64(len(data))}
}
func markdownFile(name, content string) ImportFile {
return ImportFile{Name: name, Reader: strings.NewReader(content), Size: int64(len(content))}
}
func TestImportZipAssetsAndMove(t *testing.T) {
v := newVault(t)
content := "# 标题\n\n![图](<assets/图 (1).png> \"说明\")\n\n![引用][图]\n\n[图]: assets/图%20(1).png\n\n<img src=\"assets/图%20(1).png\">\n\n[子笔记](子目录/子笔记.md)\n\n![外链](https://example.com/a.png)\n\n![缺失](assets/missing.png)\n\n`![代码](assets/图%20(1).png)`\n\n```md\n![代码块](assets/图%20(1).png)\n```\n\n> ```\n> ![引用代码](assets/图%20(1).png)\n> ```\n\n ![缩进代码](assets/图%20(1).png)\n"
source := importZip(t, zipEntry{name: "目录/笔记.md", content: content}, zipEntry{name: "目录/子目录/子笔记.md", content: "# 子笔记"}, zipEntry{name: "目录/assets/图 (1).png", content: "image"}, zipEntry{name: "未引用.txt", content: "附件"})
result, err := v.Import(context.Background(), "", "知识库", []ImportFile{source})
if err != nil {
t.Fatal(err)
}
if result.Notes != 2 || result.Assets != 3 || len(result.Warnings) != 1 || len(result.Paths) != 1 || result.Paths[0] != "知识库" {
t.Fatalf("结果不符: %+v", result)
}
body, _, err := v.Read("知识库/目录/笔记.md")
if err != nil {
t.Fatal(err)
}
link := EscapeLinkPath("笔记.assets") + "/" + assetName([]byte("image"), "x.png")
expected := strings.Replace(content, "<assets/图 (1).png>", "<"+link+">", 1)
expected = strings.Replace(expected, "[图]: assets/图%20(1).png", "[图]: "+link, 1)
expected = strings.Replace(expected, "src=\"assets/图%20(1).png\"", "src=\""+link+"\"", 1)
if string(body) != expected {
t.Fatalf("正文变更不符:\n%s", body)
}
if err := v.Mkdir("移动目标"); err != nil {
t.Fatal(err)
}
if _, err := v.Move("知识库/目录/笔记.md", "移动目标/改名.md"); err != nil {
t.Fatal(err)
}
moved, _, err := v.Read("移动目标/改名.md")
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(moved), "改名.assets/") {
t.Fatal("图片链接未跟随改名")
}
if _, err := os.Stat(filepath.Join(v.root, "移动目标/改名.assets", assetName([]byte("image"), "x.png"))); err != nil {
t.Fatal("附件未跟随", err)
}
original, err := os.ReadFile(filepath.Join(v.root, "知识库/目录/assets/图 (1).png"))
if err != nil || string(original) != "image" {
t.Fatal("原始附件丢失", err)
}
}
func TestImportRejectsInvalidWithoutPublishing(t *testing.T) {
for _, tc := range []struct {
name string
entries []zipEntry
want error
}{
{"越界", []zipEntry{{name: "../outside.md", content: "x"}}, ErrInvalidImport},
{"隐藏路径", []zipEntry{{name: ".rawmark/x.md", content: "x"}}, ErrInvalidImport},
{"重复路径", []zipEntry{{name: "a.md", content: "x"}, {name: "a.md", content: "y"}}, ErrInvalidImport},
{"符号链接", []zipEntry{{name: "a.md", content: "x", mode: os.ModeSymlink | 0o777}}, ErrInvalidImport},
{"目录文件冲突", []zipEntry{{name: "a/"}, {name: "a", content: "x"}, {name: "b.md", content: "x"}}, ErrInvalidImport},
{"文件目录冲突反向", []zipEntry{{name: "a", content: "x"}, {name: "a/"}, {name: "b.md", content: "x"}}, ErrInvalidImport},
{"无笔记", []zipEntry{{name: "a.png", content: "x"}}, ErrInvalidImport},
{"非法编码", []zipEntry{{name: "a.md", content: "\xff"}}, ErrInvalidImport},
{"附件目录内笔记", []zipEntry{{name: "a.assets/a.md", content: "x"}}, ErrInvalidImport},
{"文件目录冲突", []zipEntry{{name: "a", content: "x"}, {name: "a/b.md", content: "y"}}, ErrInvalidImport},
} {
t.Run(tc.name, func(t *testing.T) {
v := newVault(t)
_, err := v.Import(context.Background(), "", "导入", []ImportFile{importZip(t, tc.entries...)})
if !errors.Is(err, tc.want) {
t.Fatalf("错误不符: %v", err)
}
entries, err := os.ReadDir(v.root)
if err != nil {
t.Fatal(err)
}
if len(entries) != 1 || entries[0].Name() != metaDir {
t.Fatal("失败后留下可见内容或临时目录")
}
})
}
}
func TestImportBatchConflictAndCancellation(t *testing.T) {
v := newVault(t)
if _, err := v.Write("b.md", []byte("原文"), Precondition{}); err != nil {
t.Fatal(err)
}
sources := []ImportFile{markdownFile("a.md", "新文"), markdownFile("b.md", "覆盖")}
if _, err := v.Import(context.Background(), "", "", sources); !errors.Is(err, ErrExists) {
t.Fatal(err)
}
if _, _, err := v.Read("a.md"); !errors.Is(err, ErrNotFound) {
t.Fatal("批量冲突产生了半成品", err)
}
body, _, _ := v.Read("b.md")
if string(body) != "原文" {
t.Fatal("原文件被覆盖")
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := v.Import(ctx, "", "", sources[:1]); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
if _, err := v.Import(context.Background(), "不存在", "", sources[:1]); !errors.Is(err, ErrNotFound) {
t.Fatal(err)
}
result, err := v.Import(context.Background(), "", "", []ImportFile{markdownFile("a.md", "\xef\xbb\xbf正文"), markdownFile("c.md", "![缺图](local.png)")})
if err != nil || result.Notes != 2 || len(result.Warnings) != 1 {
t.Fatalf("结果不符: %+v %v", result, err)
}
body, _, _ = v.Read("a.md")
if string(body) != "正文" {
t.Fatal("UTF-8 BOM 未处理")
}
}
func TestImportLimitsAndCorruptZip(t *testing.T) {
v := newVault(t)
source := markdownFile("large.md", strings.Repeat("a", importNoteLimit+1))
if _, err := v.Import(context.Background(), "", "", []ImportFile{source}); !errors.Is(err, ErrTooLarge) {
t.Fatal(err)
}
source.Size = MaxImportBytes + 1
if _, err := v.Import(context.Background(), "", "", []ImportFile{source}); !errors.Is(err, ErrTooLarge) {
t.Fatal(err)
}
broken := markdownFile("broken.zip", "not zip")
if _, err := v.Import(context.Background(), "", "导入", []ImportFile{broken}); !errors.Is(err, ErrInvalidImport) {
t.Fatal(err)
}
}
func TestImportCorruptZipContent(t *testing.T) {
v := newVault(t)
var buffer bytes.Buffer
writer := zip.NewWriter(&buffer)
f, err := writer.CreateHeader(&zip.FileHeader{Name: "n.md", Method: zip.Store})
if err != nil {
t.Fatal(err)
}
payload := []byte("unique markdown payload")
if _, err := f.Write(payload); err != nil {
t.Fatal(err)
}
if err := writer.Close(); err != nil {
t.Fatal(err)
}
data := buffer.Bytes()
offset := bytes.Index(data, payload)
if offset < 0 {
t.Fatal("找不到测试负载")
}
data[offset] ^= 1
_, err = v.Import(context.Background(), "", "损坏", []ImportFile{{Name: "broken.zip", Reader: bytes.NewReader(data), Size: int64(len(data))}})
if !errors.Is(err, ErrInvalidImport) {
t.Fatalf("应拒绝 CRC 损坏: %v", err)
}
if _, err := os.Stat(filepath.Join(v.root, "损坏")); !os.IsNotExist(err) {
t.Fatal("损坏内容被发布", err)
}
}
+13 -6
View File
@@ -16,6 +16,14 @@ func RewriteAssetLinks(content []byte, oldBase, newBase string) []byte {
refDef := regexp.MustCompile(`^( {0,3}\[[^\]]+\]:\s*<?)` + target)
repl := []byte("${1}${2}" + strings.ReplaceAll(EscapeLinkPath(newBase+AssetsExt), "$", "$$") + "/")
return transformMarkdownLines(content, refDef, func(line []byte) []byte {
line = refDef.ReplaceAll(line, repl)
return transformOutsideCode(line, func(text []byte) []byte { return inline.ReplaceAll(text, repl) })
})
}
// transformMarkdownLines 复用链接改写的容器与代码块识别,避免导入时修改代码示例。
func transformMarkdownLines(content []byte, refDef *regexp.Regexp, transform func([]byte) []byte) []byte {
lines := bytes.SplitAfter(content, []byte("\n"))
var fence []byte // 当前所在围栏代码块的围栏字符串,nil 表示不在代码块中
var containers []linkContainer
@@ -111,8 +119,7 @@ func RewriteAssetLinks(content []byte, oldBase, newBase string) []byte {
continue
}
paragraph = !paragraphBoundary.Match(trimmed) && !refDef.Match(trimmed)
line = refDef.ReplaceAll(line, repl)
lines[i] = replaceOutsideCode(line, inline, repl)
lines[i] = transform(line)
}
return bytes.Join(lines, nil)
}
@@ -161,8 +168,8 @@ func fenceOpen(line []byte) []byte {
return rest[:n]
}
// replaceOutsideCode 只在行内代码 `…` 之外的片段上做替换
func replaceOutsideCode(line []byte, re *regexp.Regexp, repl []byte) []byte {
// transformOutsideCode 只在行内代码 `…` 之外的片段上做转换。
func transformOutsideCode(line []byte, transform func([]byte) []byte) []byte {
var out []byte
for len(line) > 0 {
start := bytes.IndexByte(line, '`')
@@ -178,9 +185,9 @@ func replaceOutsideCode(line []byte, re *regexp.Regexp, repl []byte) []byte {
break
}
codeEnd := start + n + end + n
out = append(out, re.ReplaceAll(line[:start], repl)...)
out = append(out, transform(line[:start])...)
out = append(out, line[start:codeEnd]...)
line = line[codeEnd:]
}
return append(out, re.ReplaceAll(line, repl)...)
return append(out, transform(line)...)
}
+10 -6
View File
@@ -696,12 +696,7 @@ func (v *Vault) SaveAsset(note, origName string, r io.Reader, limit int64) (stri
if int64(len(data)) > limit {
return "", ErrTooLarge
}
ext := strings.ToLower(filepath.Ext(origName))
if !extPattern.MatchString(ext) {
ext = sniffExt(data)
}
sum := sha256.Sum256(data)
name := hex.EncodeToString(sum[:8]) + ext
name := assetName(data, origName)
dir := AssetsDir(p)
v.mu.Lock()
defer v.mu.Unlock()
@@ -722,6 +717,15 @@ func (v *Vault) SaveAsset(note, origName string, r io.Reader, limit int64) (stri
return EscapeLinkPath(path.Base(AssetsDir(note))) + "/" + name, nil
}
func assetName(data []byte, origName string) string {
ext := strings.ToLower(filepath.Ext(origName))
if !extPattern.MatchString(ext) {
ext = sniffExt(data)
}
sum := sha256.Sum256(data)
return hex.EncodeToString(sum[:8]) + ext
}
func sniffExt(data []byte) string {
switch http.DetectContentType(data) {
case "image/png":