From 34d2acbbd59333c9f5b7feb88365d9acb69209c5 Mon Sep 17 00:00:00 2001 From: eezd Date: Wed, 21 Oct 2026 12:56:38 +0800 Subject: [PATCH] Add Markdown and ZIP notebook import --- README.md | 1 + internal/api/api.go | 3 +- internal/api/import_notes.go | 44 +++++ internal/api/import_notes_test.go | 117 ++++++++++++ internal/api/openapi.json | 96 ++++++++++ internal/vault/import.go | 289 ++++++++++++++++++++++++++++++ internal/vault/import_links.go | 211 ++++++++++++++++++++++ internal/vault/import_test.go | 204 +++++++++++++++++++++ internal/vault/links.go | 19 +- internal/vault/vault.go | 16 +- web/src/app/Icon.vue | 1 + web/src/app/ImportDialog.vue | 114 ++++++++++++ web/src/app/Sidebar.vue | 1 + web/src/app/TreeNode.vue | 6 +- web/src/app/Workspace.vue | 2 + web/src/app/api.ts | 15 ++ web/src/app/workspace.test.ts | 29 ++- web/src/app/workspace.ts | 14 ++ 18 files changed, 1166 insertions(+), 16 deletions(-) create mode 100644 internal/api/import_notes.go create mode 100644 internal/api/import_notes_test.go create mode 100644 internal/vault/import.go create mode 100644 internal/vault/import_links.go create mode 100644 internal/vault/import_test.go create mode 100644 web/src/app/ImportDialog.vue diff --git a/README.md b/README.md index 8b4eeb3..adecf42 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ docker run -d --name rawmark \ - `--user` 让容器以宿主机用户身份读写数据目录；不指定时容器以 UID 65532 运行，需要该用户对目录有写权限。 - 请通过 HTTPS 反向代理（Caddy、Nginx 等）访问，并转发 WebSocket（`/api/v1/events`）。 - 分享：命令面板（F1）中「分享当前笔记」生成只读链接 `/s/…`，可设访问密码与有效期；「管理分享链接」查看与撤销。 +- 导入：侧栏顶部或文件夹右键菜单选择「导入」，支持多选 UTF-8 `.md` 文件或单个 ZIP（总上传上限 100 MB）。ZIP 新建同名文件夹并保留层级，本地图片转换为笔记独立附件，原附件保留；同名内容不覆盖，缺失的本地链接会提示，外部图片不会自动下载。 - 自动化：「API 令牌」中创建只读或读写令牌，以 `Authorization: Bearer <令牌>` 调用 REST API，接口说明见 `/api/v1/openapi.json`。 | 环境变量 | 默认值 | 说明 | diff --git a/internal/api/api.go b/internal/api/api.go index 69e9472..e8155ee 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -84,6 +84,7 @@ func (s *Server) routes() []route { {"DELETE /api/v1/files/{path...}", authed, s.deleteFile}, {"POST /api/v1/move", authed, s.move}, {"POST /api/v1/folders", authed, s.mkdir}, + {"POST /api/v1/import", authed, s.importNotes}, {"POST /api/v1/assets", authed, s.uploadAsset}, {"POST /api/v1/assets/import", authed, s.importAsset}, {"GET /api/v1/raw/{path...}", authed, s.raw}, @@ -492,7 +493,7 @@ func vaultError(w http.ResponseWriter, r *http.Request, err error) { switch { case errors.As(err, &conflict): writeJSON(w, 412, map[string]string{"error": err.Error(), "rev": conflict.Rev}) - case errors.Is(err, vault.ErrInvalidPath): + case errors.Is(err, vault.ErrInvalidPath), errors.Is(err, vault.ErrInvalidImport): writeError(w, 400, err.Error()) case errors.Is(err, vault.ErrNotFound): writeError(w, 404, err.Error()) diff --git a/internal/api/import_notes.go b/internal/api/import_notes.go new file mode 100644 index 0000000..7fca4b7 --- /dev/null +++ b/internal/api/import_notes.go @@ -0,0 +1,44 @@ +package api + +import ( + "errors" + "net/http" + + "rawmark/internal/vault" +) + +func (s *Server) importNotes(w http.ResponseWriter, r *http.Request) { + // 总请求上限包含表单开销，文件总量另由 vault 校验。 + r.Body = http.MaxBytesReader(w, r.Body, vault.MaxImportBytes+(1<<20)) + if err := r.ParseMultipartForm(8 << 20); err != nil { + var limit *http.MaxBytesError + if errors.As(err, &limit) { + writeError(w, 413, "导入文件总量不能超过 100 MB") + } else { + writeError(w, 400, "无法读取导入表单") + } + return + } + defer r.MultipartForm.RemoveAll() + headers := r.MultipartForm.File["files"] + if len(headers) == 0 || len(headers) > 10_000 { + writeError(w, 400, "请选择 Markdown 文件或一个 ZIP") + return + } + files := make([]vault.ImportFile, 0, len(headers)) + for _, h := range headers { + f, err := h.Open() + if err != nil { + internalError(w, r, err) + return + } + defer f.Close() + files = append(files, vault.ImportFile{Name: h.Filename, Reader: f, Size: h.Size}) + } + result, err := s.vault.Import(r.Context(), r.FormValue("dir"), r.FormValue("folder"), files) + if err != nil { + vaultError(w, r, err) + return + } + writeJSON(w, http.StatusCreated, result) +} diff --git a/internal/api/import_notes_test.go b/internal/api/import_notes_test.go new file mode 100644 index 0000000..bff4301 --- /dev/null +++ b/internal/api/import_notes_test.go @@ -0,0 +1,117 @@ +package api + +import ( + "archive/zip" + "bytes" + "encoding/json" + "io" + "mime/multipart" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + + "rawmark/internal/vault" +) + +func TestImportNotesAPI(t *testing.T) { + f := newFixture(t) + send := func(client *http.Client, token, dir, folder, name string, data []byte) *http.Response { + t.Helper() + var body bytes.Buffer + form := multipart.NewWriter(&body) + if err := form.WriteField("dir", dir); err != nil { + t.Fatal(err) + } + if err := form.WriteField("folder", folder); err != nil { + t.Fatal(err) + } + file, err := form.CreateFormFile("files", name) + if err != nil { + t.Fatal(err) + } + if _, err := file.Write(data); err != nil { + t.Fatal(err) + } + if err := form.Close(); err != nil { + t.Fatal(err) + } + req, err := http.NewRequest("POST", f.url+"/api/v1/import", &body) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Content-Type", form.FormDataContentType()) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + res, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + return res + } + readToken := decode[struct{ Token string }](t, f.do(f.admin, "POST", "/api/v1/tokens", `{"name":"只读","scope":"read"}`, nil)).Token + for _, tc := range []struct { + client *http.Client + token string + status int + }{{newClient(), "", 401}, {newClient(), readToken, 403}} { + res := send(tc.client, tc.token, "", "", "n.md", []byte("正文")) + res.Body.Close() + if res.StatusCode != tc.status { + t.Fatalf("权限状态不符: %d", res.StatusCode) + } + } + res := send(f.admin, "", "", "", "n.md", []byte("正文")) + if res.StatusCode != 201 { + t.Fatalf("导入失败: %d", res.StatusCode) + } + var result vault.ImportResult + if err := json.NewDecoder(res.Body).Decode(&result); err != nil { + t.Fatal(err) + } + res.Body.Close() + if result.Notes != 1 || len(result.Paths) != 1 || result.Paths[0] != "n.md" || result.Warnings == nil { + t.Fatalf("响应不符: %+v", result) + } + res = send(f.admin, "", "", "", "n.md", []byte("覆盖")) + res.Body.Close() + if res.StatusCode != 409 { + t.Fatalf("应拒绝冲突: %d", res.StatusCode) + } + data, err := os.ReadFile(filepath.Join(f.root, "n.md")) + if err != nil || string(data) != "正文" { + t.Fatal("原文被覆盖", err) + } + writeToken := decode[struct{ Token string }](t, f.do(f.admin, "POST", "/api/v1/tokens", `{"name":"写入","scope":"write"}`, nil)).Token + var b bytes.Buffer + zw := zip.NewWriter(&b) + zf, err := zw.Create("子目录/笔记.md") + if err != nil { + t.Fatal(err) + } + if _, err := io.WriteString(zf, "# ZIP"); err != nil { + t.Fatal(err) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + res = send(newClient(), writeToken, "", "知识库", "export.md.zip", b.Bytes()) + res.Body.Close() + if res.StatusCode != 201 { + t.Fatalf("写入令牌导入失败: %d", res.StatusCode) + } + for _, tc := range []struct { + dir, folder, name, data string + status int + }{ + {"", "", "x.zip", "invalid", 400}, {"不存在", "", "x.md", "x", 404}, {"n.md", "", "x.md", "x", 400}, {"", "", "x.md", strings.Repeat("a", (20<<20)+1), 413}, + } { + res := send(f.admin, "", tc.dir, tc.folder, tc.name, []byte(tc.data)) + res.Body.Close() + if res.StatusCode != tc.status { + t.Fatalf("%s 状态不符: %d", tc.name, res.StatusCode) + } + } +} diff --git a/internal/api/openapi.json b/internal/api/openapi.json index 08719cb..2b65039 100644 --- a/internal/api/openapi.json +++ b/internal/api/openapi.json @@ -2180,6 +2180,102 @@ } ] } + }, + "/import": { + "post": { + "summary": "导入 Markdown 文件或 ZIP 知识库", + "tags": [ + "文件" + ], + "description": "多选 .md 或单个 ZIP，总上传不超过 100 MB；ZIP 必须指定新建文件夹名称。目标目录必须存在，冲突不覆盖。解压及整理后上限 512 MiB、10,000 项，单篇笔记 20 MiB、附件 50 MiB。Markdown 与 ZIP 文件名须为 UTF-8。ZIP 层级保留，本地图片生成笔记独立附件，原附件保留；外部链接不下载，缺失本地目标以警告返回。", + "requestBody": { + "required": true, + "content": { + "multipart/form-data": { + "schema": { + "type": "object", + "required": [ + "files" + ], + "properties": { + "dir": { + "type": "string", + "description": "目标目录，空字符串为根目录" + }, + "folder": { + "type": "string", + "description": "ZIP 的新建文件夹名称；普通 Markdown 留空" + }, + "files": { + "type": "array", + "items": { + "type": "string", + "format": "binary" + } + } + } + } + } + } + }, + "responses": { + "201": { + "description": "导入成功", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "paths", + "notes", + "assets", + "warnings" + ], + "properties": { + "paths": { + "type": "array", + "items": { + "type": "string" + } + }, + "notes": { + "type": "integer" + }, + "assets": { + "type": "integer", + "description": "附件文件总数，包含原始附件和独立副本" + }, + "warnings": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + }, + "400": { + "description": "格式、路径或编码无效" + }, + "401": { + "description": "未登录或令牌无效" + }, + "403": { + "description": "只读令牌不能导入" + }, + "404": { + "description": "目标目录不存在" + }, + "409": { + "description": "目标已存在，不覆盖" + }, + "413": { + "description": "超过上传、解压、文件数量或单文件大小限制" + } + } + } } }, "components": { diff --git a/internal/vault/import.go b/internal/vault/import.go new file mode 100644 index 0000000..ce54e08 --- /dev/null +++ b/internal/vault/import.go @@ -0,0 +1,289 @@ +package vault + +import ( + "archive/zip" + "bytes" + "compress/flate" + "context" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "sort" + "strings" + "syscall" + "unicode/utf8" +) + +const ( + MaxImportBytes = 100_000_000 + maxImportExpanded = 512 << 20 + maxImportEntries = 10_000 + importNoteLimit = 20 << 20 + importAssetLimit = 50 << 20 +) + +var ErrInvalidImport = errors.New("导入内容无效") + +// ImportFile 的读取范围是单个上传文件，ZIP 与普通 Markdown 共用这一输入边界。 +type ImportFile struct { + Name string + Reader io.ReaderAt + Size int64 +} + +type ImportResult struct { + Paths []string `json:"paths"` + Notes int `json:"notes"` + Assets int `json:"assets"` + Warnings []string `json:"warnings"` +} + +// Import 在隐藏目录内整理完整内容，提交前统一检查冲突，绝不覆盖已有文件。 +func (v *Vault) Import(ctx context.Context, dir, folder string, files []ImportFile) (result ImportResult, err error) { + result = ImportResult{Paths: []string{}, Warnings: []string{}} + if len(files) == 0 || len(files) > maxImportEntries { + return result, fmt.Errorf("%w：请选择 Markdown 文件或一个 ZIP", ErrInvalidImport) + } + zipped := len(files) == 1 && strings.EqualFold(path.Ext(files[0].Name), ".zip") + if zipped { + if _, err := CleanPath(folder); err != nil || strings.Contains(folder, "/") || strings.HasSuffix(folder, AssetsExt) { + return result, fmt.Errorf("%w：导入文件夹名称无效", ErrInvalidImport) + } + } else if folder != "" { + return result, fmt.Errorf("%w：普通 Markdown 不使用外层文件夹", ErrInvalidImport) + } + if dir != "" { + if _, err := CleanPath(dir); err != nil { + return result, err + } + } + var uploadBytes int64 + for _, f := range files { + if f.Size < 0 || f.Size > MaxImportBytes-uploadBytes { + return result, ErrTooLarge + } + uploadBytes += f.Size + } + stage, err := os.MkdirTemp(v.root, tmpPrefix+"import-") + if err != nil { + return result, err + } + defer func() { + if cleanup := os.RemoveAll(stage); cleanup != nil { + err = errors.Join(err, fmt.Errorf("清理导入临时目录失败：%w", cleanup)) + } + }() + staged := &Vault{root: stage} + entries := map[string]struct{}{} + var total int64 + var notes []string + // 每个来源文件只写一次；目录与文件重名也会在暂存阶段报错。 + write := func(name string, isDir bool, reader io.Reader) error { + if err := ctx.Err(); err != nil { + return err + } + if _, err := CleanPath(name); err != nil || !utf8.ValidString(name) { + return fmt.Errorf("%w：ZIP 含有非法路径", ErrInvalidImport) + } + if _, exists := entries[name]; exists { + return fmt.Errorf("%w：存在重复路径", ErrInvalidImport) + } + entries[name] = struct{}{} + dest := filepath.Join(stage, filepath.FromSlash(name)) + if isDir { + err := os.MkdirAll(dest, 0o755) + if errors.Is(err, fs.ErrExist) || errors.Is(err, syscall.ENOTDIR) { + return fmt.Errorf("%w：文件与目录路径冲突", ErrInvalidImport) + } + return err + } + if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { + if errors.Is(err, syscall.ENOTDIR) || errors.Is(err, fs.ErrExist) { + return fmt.Errorf("%w：文件与目录路径冲突", ErrInvalidImport) + } + return err + } + f, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + if err != nil { + if errors.Is(err, fs.ErrExist) || errors.Is(err, syscall.ENOTDIR) { + return fmt.Errorf("%w：文件与目录路径冲突", ErrInvalidImport) + } + return err + } + limit := int64(importAssetLimit) + if IsNote(name) { + limit = importNoteLimit + notes = append(notes, name) + } + limit = min(limit, maxImportExpanded-total) + n, copyErr := io.Copy(f, io.LimitReader(reader, limit+1)) + closeErr := f.Close() + total += n + if n > limit { + return ErrTooLarge + } + return errors.Join(copyErr, closeErr) + } + if zipped { + z, err := zip.NewReader(files[0].Reader, files[0].Size) + if err != nil { + return result, fmt.Errorf("%w：无法读取 ZIP：%w", ErrInvalidImport, err) + } + if len(z.File) > maxImportEntries { + return result, ErrTooLarge + } + for _, f := range z.File { + if !f.Mode().IsRegular() && !f.Mode().IsDir() { + return result, fmt.Errorf("%w：ZIP 不允许符号链接或特殊文件", ErrInvalidImport) + } + name := strings.TrimSuffix(f.Name, "/") + r, err := f.Open() + if err != nil { + return result, fmt.Errorf("%w：无法读取 ZIP 条目：%w", ErrInvalidImport, err) + } + err = write(name, f.FileInfo().IsDir(), r) + err = errors.Join(err, r.Close()) + if err != nil { + var corrupt flate.CorruptInputError + if errors.Is(err, zip.ErrChecksum) || errors.Is(err, io.ErrUnexpectedEOF) || errors.As(err, &corrupt) { + err = fmt.Errorf("%w：ZIP 内容损坏：%w", ErrInvalidImport, err) + } + return result, err + } + } + } else { + for _, f := range files { + if !IsNote(f.Name) || strings.Contains(f.Name, "/") { + return result, fmt.Errorf("%w：只接受 .md 文件或单个 ZIP", ErrInvalidImport) + } + if err := write(f.Name, false, io.NewSectionReader(f.Reader, 0, f.Size)); err != nil { + return result, err + } + } + } + if len(notes) == 0 { + return result, fmt.Errorf("%w：没有可导入的 Markdown 笔记", ErrInvalidImport) + } + sort.Strings(notes) + for _, note := range notes { + if err := ctx.Err(); err != nil { + return result, err + } + for _, seg := range strings.Split(path.Dir(note), "/") { + if strings.HasSuffix(seg, AssetsExt) { + return result, fmt.Errorf("%w：笔记不能放在 .assets 附件目录内", ErrInvalidImport) + } + } + content, _, err := staged.Read(note) + if err != nil { + return result, err + } + if !utf8.Valid(content) || bytes.IndexByte(content, 0) >= 0 { + return result, fmt.Errorf("%w：Markdown 必须是 UTF-8 文本", ErrInvalidImport) + } + content = bytes.TrimPrefix(content, []byte{0xef, 0xbb, 0xbf}) + updated, warnings, err := staged.importLinks(ctx, note, content, &total) + if err != nil { + return result, err + } + result.Warnings = append(result.Warnings, warnings...) + if len(updated) > importNoteLimit || total+int64(len(updated)-len(content)) > maxImportExpanded { + return result, ErrTooLarge + } + total += int64(len(updated) - len(content)) + if err := os.WriteFile(filepath.Join(stage, filepath.FromSlash(note)), updated, 0o644); err != nil { + return result, err + } + } + result.Notes = len(notes) + // 附件数量包含原始附件及为各笔记生成的独立副本。 + count := 0 + err = filepath.WalkDir(stage, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + count++ + if count > maxImportEntries+1 { + return ErrTooLarge + } + if !d.IsDir() && !IsNote(d.Name()) { + result.Assets++ + } + return nil + }) + if err != nil { + return result, err + } + top, err := os.ReadDir(stage) + if err != nil { + return result, err + } + targets := []string{} + if zipped { + targets = append(targets, path.Join(dir, folder)) + } else { + for _, f := range top { + targets = append(targets, path.Join(dir, f.Name())) + } + } + v.mu.Lock() + defer v.mu.Unlock() + parent := v.root + if dir != "" { + parent, err = v.abs(dir) + if err != nil { + return result, err + } + } + info, err := os.Stat(parent) + if errors.Is(err, fs.ErrNotExist) { + return result, ErrNotFound + } + if err != nil { + return result, err + } + if !info.IsDir() { + return result, fmt.Errorf("%w：目标不是文件夹", ErrInvalidImport) + } + for _, target := range targets { + dest, err := v.abs(target) + if err != nil { + return result, err + } + if _, err := os.Lstat(dest); err == nil { + return result, fmt.Errorf("%w：%s，请修改名称", ErrExists, target) + } else if !errors.Is(err, fs.ErrNotExist) { + return result, err + } + } + if err := ctx.Err(); err != nil { + return result, err + } + if zipped { + // 暂存目录与仓库位于同一文件系统，整个 ZIP 一次 rename 发布。 + err = os.Rename(stage, filepath.Join(parent, folder)) + } else { + var published []string + for _, f := range top { + name := f.Name() + if err = os.Rename(filepath.Join(stage, name), filepath.Join(parent, name)); err != nil { + for _, prev := range published { + if rollback := os.Rename(filepath.Join(parent, prev), filepath.Join(stage, prev)); rollback != nil { + err = errors.Join(err, fmt.Errorf("回滚导入失败：%w", rollback)) + } + } + break + } + published = append(published, name) + } + } + if err != nil { + return result, err + } + result.Paths = targets + return result, nil +} diff --git a/internal/vault/import_links.go b/internal/vault/import_links.go new file mode 100644 index 0000000..a6f5c3a --- /dev/null +++ b/internal/vault/import_links.go @@ -0,0 +1,211 @@ +package vault + +import ( + "bytes" + "context" + "fmt" + "html" + "net/url" + "os" + "path" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" +) + +var importRef = regexp.MustCompile(`^ {0,3}\[[^\]\r\n]+\]:[ \t]*`) +var importSrc = regexp.MustCompile(`(?i)<(?:img|source|video|audio)\b[^>]*?\bsrc\s*=\s*["']([^"']*)["']`) +var markdownEscape = regexp.MustCompile("\\\\([!\"#$%&'()*+,\\-./:;<=>?@\\[\\]\\\\^_`{|}~])") + +type importLink struct { + start, end int + html bool +} + +// linkDestination 返回目的地址本身的区间，不含尖括号、标题或闭合括号。 +func linkDestination(text []byte, start int) (int, int) { + for start < len(text) && (text[start] == ' ' || text[start] == '\t') { + start++ + } + if start == len(text) { + return start, start + } + if text[start] == '<' { + end := start + 1 + for end < len(text) && text[end] != '>' && text[end] != '\n' && text[end] != '\r' { + if text[end] == '\\' && end+1 < len(text) { + end++ + } + end++ + } + if end < len(text) && text[end] == '>' { + return start + 1, end + } + return start, start + } + depth := 0 + end := start + for end < len(text) { + c := text[end] + if c == '\\' && end+1 < len(text) { + end += 2 + continue + } + if c == '(' { + depth++ + } + if c == ')' { + if depth == 0 { + break + } + depth-- + } + if c == ' ' || c == '\t' || c == '\n' || c == '\r' { + break + } + end++ + } + if depth != 0 { + return start, start + } + return start, end +} + +func importLinkSpans(text []byte) []importLink { + var spans []importLink + if m := importRef.FindIndex(text); m != nil { + start, end := linkDestination(text, m[1]) + if end > start { + spans = append(spans, importLink{start, end, false}) + } + } + for pos := 0; pos < len(text); { + i := bytes.Index(text[pos:], []byte("](")) + if i < 0 { + break + } + i += pos + // 转义的闭括号不是链接语法。 + escapes := 0 + for j := i - 1; j >= 0 && text[j] == '\\'; j-- { + escapes++ + } + start, end := linkDestination(text, i+2) + if escapes%2 == 0 && end > start { + spans = append(spans, importLink{start, end, false}) + } + pos = max(i+2, end) + } + for _, m := range importSrc.FindAllSubmatchIndex(text, -1) { + spans = append(spans, importLink{m[2], m[3], true}) + } + return spans +} + +func (v *Vault) importLinks(ctx context.Context, note string, content []byte, total *int64) ([]byte, []string, error) { + var failure error + missing := map[string]bool{} + replacements := map[string]string{} + transform := func(text []byte) []byte { + spans := importLinkSpans(text) + // 区间可能来自 HTML 或 Markdown，按原始顺序替换，保持其他字节不变。 + sort.Slice(spans, func(i, j int) bool { return spans[i].start < spans[j].start }) + var out []byte + last := 0 + for _, span := range spans { + if span.start < last || failure != nil { + continue + } + raw := string(text[span.start:span.end]) + target := raw + if span.html { + target = html.UnescapeString(target) + } else { + target = markdownEscape.ReplaceAllString(target, "$1") + } + u, err := url.Parse(target) + if err != nil || u.IsAbs() || u.Host != "" || strings.HasPrefix(target, "/") || u.Path == "" { + continue + } + rel := path.Clean(path.Join(path.Dir(note), u.Path)) + if _, err := CleanPath(rel); err != nil { + missing[rel] = true + continue + } + info, err := os.Stat(filepath.Join(v.root, filepath.FromSlash(rel))) + if os.IsNotExist(err) { + missing[rel] = true + continue + } + if err != nil { + failure = err + continue + } + if !info.Mode().IsRegular() { + continue + } + switch strings.ToLower(path.Ext(rel)) { + case ".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".avif", ".apng", ".bmp", ".ico", ".tif", ".tiff": + default: + continue + } + replacement, found := replacements[rel] + if !found { + if err := ctx.Err(); err != nil { + failure = err + continue + } + data, err := os.ReadFile(filepath.Join(v.root, filepath.FromSlash(rel))) + if err != nil { + failure = err + continue + } + // 只统计新生成的副本；已有同名附件必须与哈希对应的内容一致。 + dest := filepath.Join(v.root, filepath.FromSlash(AssetsDir(note)), assetName(data, path.Base(rel))) + existing, readErr := os.ReadFile(dest) + if readErr != nil && !os.IsNotExist(readErr) { + failure = readErr + continue + } + if readErr == nil && !bytes.Equal(data, existing) { + failure = fmt.Errorf("%w：附件名称与内容冲突", ErrInvalidImport) + continue + } + if os.IsNotExist(readErr) { + if int64(len(data)) > maxImportExpanded-*total { + failure = ErrTooLarge + continue + } + *total += int64(len(data)) + } + replacement, err = v.SaveAsset(note, path.Base(rel), bytes.NewReader(data), importAssetLimit) + if err != nil { + failure = err + continue + } + replacements[rel] = replacement + } + if u.RawQuery != "" { + replacement += "?" + u.RawQuery + } + if u.Fragment != "" { + replacement += "#" + u.EscapedFragment() + } + if span.html { + replacement = html.EscapeString(replacement) + } + out = append(out, text[last:span.start]...) + out = append(out, replacement...) + last = span.end + } + return append(out, text[last:]...) + } + updated := transformMarkdownLines(content, importRef, func(line []byte) []byte { return transformOutsideCode(line, transform) }) + warnings := []string{} + if len(missing) > 0 { + warnings = append(warnings, note+"："+strconv.Itoa(len(missing))+" 个本地链接的目标不在导入内容中，已保留原链接") + } + return updated, warnings, failure +} diff --git a/internal/vault/import_test.go b/internal/vault/import_test.go new file mode 100644 index 0000000..e62fc97 --- /dev/null +++ b/internal/vault/import_test.go @@ -0,0 +1,204 @@ +package vault + +import ( + "archive/zip" + "bytes" + "context" + "errors" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" +) + +type zipEntry struct { + name, content string + mode fs.FileMode +} + +func importZip(t *testing.T, entries ...zipEntry) ImportFile { + t.Helper() + var b bytes.Buffer + w := zip.NewWriter(&b) + for _, e := range entries { + h := &zip.FileHeader{Name: e.name, Method: zip.Deflate} + if e.mode != 0 { + h.SetMode(e.mode) + } + f, err := w.CreateHeader(h) + if err != nil { + t.Fatal(err) + } + if _, err := f.Write([]byte(e.content)); err != nil { + t.Fatal(err) + } + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + data := b.Bytes() + return ImportFile{Name: "知识库.md.zip", Reader: bytes.NewReader(data), Size: int64(len(data))} +} + +func markdownFile(name, content string) ImportFile { + return ImportFile{Name: name, Reader: strings.NewReader(content), Size: int64(len(content))} +} + +func TestImportZipAssetsAndMove(t *testing.T) { + v := newVault(t) + content := "# 标题\n\n![图]( \"说明\")\n\n![引用][图]\n\n[图]: assets/图%20(1).png\n\n\n\n[子笔记](子目录/子笔记.md)\n\n![外链](https://example.com/a.png)\n\n![缺失](assets/missing.png)\n\n`![代码](assets/图%20(1).png)`\n\n```md\n![代码块](assets/图%20(1).png)\n```\n\n> ```\n> ![引用代码](assets/图%20(1).png)\n> ```\n\n ![缩进代码](assets/图%20(1).png)\n" + source := importZip(t, zipEntry{name: "目录/笔记.md", content: content}, zipEntry{name: "目录/子目录/子笔记.md", content: "# 子笔记"}, zipEntry{name: "目录/assets/图 (1).png", content: "image"}, zipEntry{name: "未引用.txt", content: "附件"}) + result, err := v.Import(context.Background(), "", "知识库", []ImportFile{source}) + if err != nil { + t.Fatal(err) + } + if result.Notes != 2 || result.Assets != 3 || len(result.Warnings) != 1 || len(result.Paths) != 1 || result.Paths[0] != "知识库" { + t.Fatalf("结果不符: %+v", result) + } + body, _, err := v.Read("知识库/目录/笔记.md") + if err != nil { + t.Fatal(err) + } + link := EscapeLinkPath("笔记.assets") + "/" + assetName([]byte("image"), "x.png") + expected := strings.Replace(content, "", "<"+link+">", 1) + expected = strings.Replace(expected, "[图]: assets/图%20(1).png", "[图]: "+link, 1) + expected = strings.Replace(expected, "src=\"assets/图%20(1).png\"", "src=\""+link+"\"", 1) + if string(body) != expected { + t.Fatalf("正文变更不符:\n%s", body) + } + if err := v.Mkdir("移动目标"); err != nil { + t.Fatal(err) + } + if _, err := v.Move("知识库/目录/笔记.md", "移动目标/改名.md"); err != nil { + t.Fatal(err) + } + moved, _, err := v.Read("移动目标/改名.md") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(moved), "改名.assets/") { + t.Fatal("图片链接未跟随改名") + } + if _, err := os.Stat(filepath.Join(v.root, "移动目标/改名.assets", assetName([]byte("image"), "x.png"))); err != nil { + t.Fatal("附件未跟随", err) + } + original, err := os.ReadFile(filepath.Join(v.root, "知识库/目录/assets/图 (1).png")) + if err != nil || string(original) != "image" { + t.Fatal("原始附件丢失", err) + } +} + +func TestImportRejectsInvalidWithoutPublishing(t *testing.T) { + for _, tc := range []struct { + name string + entries []zipEntry + want error + }{ + {"越界", []zipEntry{{name: "../outside.md", content: "x"}}, ErrInvalidImport}, + {"隐藏路径", []zipEntry{{name: ".rawmark/x.md", content: "x"}}, ErrInvalidImport}, + {"重复路径", []zipEntry{{name: "a.md", content: "x"}, {name: "a.md", content: "y"}}, ErrInvalidImport}, + {"符号链接", []zipEntry{{name: "a.md", content: "x", mode: os.ModeSymlink | 0o777}}, ErrInvalidImport}, + {"目录文件冲突", []zipEntry{{name: "a/"}, {name: "a", content: "x"}, {name: "b.md", content: "x"}}, ErrInvalidImport}, + {"文件目录冲突反向", []zipEntry{{name: "a", content: "x"}, {name: "a/"}, {name: "b.md", content: "x"}}, ErrInvalidImport}, + {"无笔记", []zipEntry{{name: "a.png", content: "x"}}, ErrInvalidImport}, + {"非法编码", []zipEntry{{name: "a.md", content: "\xff"}}, ErrInvalidImport}, + {"附件目录内笔记", []zipEntry{{name: "a.assets/a.md", content: "x"}}, ErrInvalidImport}, + {"文件目录冲突", []zipEntry{{name: "a", content: "x"}, {name: "a/b.md", content: "y"}}, ErrInvalidImport}, + } { + t.Run(tc.name, func(t *testing.T) { + v := newVault(t) + _, err := v.Import(context.Background(), "", "导入", []ImportFile{importZip(t, tc.entries...)}) + if !errors.Is(err, tc.want) { + t.Fatalf("错误不符: %v", err) + } + entries, err := os.ReadDir(v.root) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 || entries[0].Name() != metaDir { + t.Fatal("失败后留下可见内容或临时目录") + } + }) + } +} + +func TestImportBatchConflictAndCancellation(t *testing.T) { + v := newVault(t) + if _, err := v.Write("b.md", []byte("原文"), Precondition{}); err != nil { + t.Fatal(err) + } + sources := []ImportFile{markdownFile("a.md", "新文"), markdownFile("b.md", "覆盖")} + if _, err := v.Import(context.Background(), "", "", sources); !errors.Is(err, ErrExists) { + t.Fatal(err) + } + if _, _, err := v.Read("a.md"); !errors.Is(err, ErrNotFound) { + t.Fatal("批量冲突产生了半成品", err) + } + body, _, _ := v.Read("b.md") + if string(body) != "原文" { + t.Fatal("原文件被覆盖") + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := v.Import(ctx, "", "", sources[:1]); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + if _, err := v.Import(context.Background(), "不存在", "", sources[:1]); !errors.Is(err, ErrNotFound) { + t.Fatal(err) + } + result, err := v.Import(context.Background(), "", "", []ImportFile{markdownFile("a.md", "\xef\xbb\xbf正文"), markdownFile("c.md", "![缺图](local.png)")}) + if err != nil || result.Notes != 2 || len(result.Warnings) != 1 { + t.Fatalf("结果不符: %+v %v", result, err) + } + body, _, _ = v.Read("a.md") + if string(body) != "正文" { + t.Fatal("UTF-8 BOM 未处理") + } +} + +func TestImportLimitsAndCorruptZip(t *testing.T) { + v := newVault(t) + source := markdownFile("large.md", strings.Repeat("a", importNoteLimit+1)) + if _, err := v.Import(context.Background(), "", "", []ImportFile{source}); !errors.Is(err, ErrTooLarge) { + t.Fatal(err) + } + source.Size = MaxImportBytes + 1 + if _, err := v.Import(context.Background(), "", "", []ImportFile{source}); !errors.Is(err, ErrTooLarge) { + t.Fatal(err) + } + broken := markdownFile("broken.zip", "not zip") + if _, err := v.Import(context.Background(), "", "导入", []ImportFile{broken}); !errors.Is(err, ErrInvalidImport) { + t.Fatal(err) + } +} + +func TestImportCorruptZipContent(t *testing.T) { + v := newVault(t) + var buffer bytes.Buffer + writer := zip.NewWriter(&buffer) + f, err := writer.CreateHeader(&zip.FileHeader{Name: "n.md", Method: zip.Store}) + if err != nil { + t.Fatal(err) + } + payload := []byte("unique markdown payload") + if _, err := f.Write(payload); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + data := buffer.Bytes() + offset := bytes.Index(data, payload) + if offset < 0 { + t.Fatal("找不到测试负载") + } + data[offset] ^= 1 + _, err = v.Import(context.Background(), "", "损坏", []ImportFile{{Name: "broken.zip", Reader: bytes.NewReader(data), Size: int64(len(data))}}) + if !errors.Is(err, ErrInvalidImport) { + t.Fatalf("应拒绝 CRC 损坏: %v", err) + } + if _, err := os.Stat(filepath.Join(v.root, "损坏")); !os.IsNotExist(err) { + t.Fatal("损坏内容被发布", err) + } +} diff --git a/internal/vault/links.go b/internal/vault/links.go index bd8caf6..beeea7e 100644 --- a/internal/vault/links.go +++ b/internal/vault/links.go @@ -16,6 +16,14 @@ func RewriteAssetLinks(content []byte, oldBase, newBase string) []byte { refDef := regexp.MustCompile(`^( {0,3}\[[^\]]+\]:\s* 0 { start := bytes.IndexByte(line, '`') @@ -178,9 +185,9 @@ func replaceOutsideCode(line []byte, re *regexp.Regexp, repl []byte) []byte { break } codeEnd := start + n + end + n - out = append(out, re.ReplaceAll(line[:start], repl)...) + out = append(out, transform(line[:start])...) out = append(out, line[start:codeEnd]...) line = line[codeEnd:] } - return append(out, re.ReplaceAll(line, repl)...) + return append(out, transform(line)...) } diff --git a/internal/vault/vault.go b/internal/vault/vault.go index f037501..b7cea67 100644 --- a/internal/vault/vault.go +++ b/internal/vault/vault.go @@ -696,12 +696,7 @@ func (v *Vault) SaveAsset(note, origName string, r io.Reader, limit int64) (stri if int64(len(data)) > limit { return "", ErrTooLarge } - ext := strings.ToLower(filepath.Ext(origName)) - if !extPattern.MatchString(ext) { - ext = sniffExt(data) - } - sum := sha256.Sum256(data) - name := hex.EncodeToString(sum[:8]) + ext + name := assetName(data, origName) dir := AssetsDir(p) v.mu.Lock() defer v.mu.Unlock() @@ -722,6 +717,15 @@ func (v *Vault) SaveAsset(note, origName string, r io.Reader, limit int64) (stri return EscapeLinkPath(path.Base(AssetsDir(note))) + "/" + name, nil } +func assetName(data []byte, origName string) string { + ext := strings.ToLower(filepath.Ext(origName)) + if !extPattern.MatchString(ext) { + ext = sniffExt(data) + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:8]) + ext +} + func sniffExt(data []byte) string { switch http.DetectContentType(data) { case "image/png": diff --git a/web/src/app/Icon.vue b/web/src/app/Icon.vue index 763941a..e2f684c 100644 --- a/web/src/app/Icon.vue +++ b/web/src/app/Icon.vue @@ -3,6 +3,7 @@ const paths = { menu: 'M4 6h16M4 12h16M4 18h16', noteAdd: 'M14 3H6a2 2 0 0 0-2 2v14a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V9zM14 3v6h6M12 12v6M9 15h6', + import: 'M12 3v12m-5-5 5 5 5-5M4 15v5h16v-5', folderAdd: 'M3 7a2 2 0 0 1 2-2h4l2 2h8a2 2 0 0 1 2 2v8a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2zM12 10v6M9 13h6', folder: 'M3 7a2 2 0 0 1 2-2h4l2 2h8a2 2 0 0 1 2 2v8a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2z', note: 'M14 3H6a2 2 0 0 0-2 2v14a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V9zM14 3v6h6', diff --git a/web/src/app/ImportDialog.vue b/web/src/app/ImportDialog.vue new file mode 100644 index 0000000..0bed2d8 --- /dev/null +++ b/web/src/app/ImportDialog.vue @@ -0,0 +1,114 @@ + + + + + diff --git a/web/src/app/Sidebar.vue b/web/src/app/Sidebar.vue index 089d0e2..f70f5ec 100644 --- a/web/src/app/Sidebar.vue +++ b/web/src/app/Sidebar.vue @@ -39,6 +39,7 @@ function onRootDrop(e: DragEvent) { +