Convert pasted web content to Markdown and import remote images
This commit is contained in:
eezd committed
1 parent
84e89c8fe4
commit
8c13dfeaec
13 files changed
+820
-15
No files matched your search
@@ -85,6 +85,7 @@ func (s *Server) routes() []route {
|
||||
{"POST /api/v1/move", authed, s.move},
|
||||
{"POST /api/v1/folders", authed, s.mkdir},
|
||||
{"POST /api/v1/assets", authed, s.uploadAsset},
|
||||
{"POST /api/v1/assets/import", authed, s.importAsset},
|
||||
{"GET /api/v1/raw/{path...}", authed, s.raw},
|
||||
{"GET /api/v1/trash", authed, s.trashList},
|
||||
{"POST /api/v1/trash/{id}/restore", authed, s.trashRestore},
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"rawmark/internal/vault"
|
||||
)
|
||||
|
||||
var (
|
||||
errImageAddress = errors.New("图片地址必须是公网 HTTP 或 HTTPS 地址")
|
||||
errImageType = errors.New("仅支持导入 PNG、JPEG、GIF 和 WebP 图片")
|
||||
)
|
||||
|
||||
var blockedImageNetworks = []netip.Prefix{
|
||||
netip.MustParsePrefix("0.0.0.0/8"),
|
||||
netip.MustParsePrefix("100.64.0.0/10"),
|
||||
netip.MustParsePrefix("192.0.0.0/24"),
|
||||
netip.MustParsePrefix("192.0.2.0/24"),
|
||||
netip.MustParsePrefix("198.18.0.0/15"),
|
||||
netip.MustParsePrefix("198.51.100.0/24"),
|
||||
netip.MustParsePrefix("203.0.113.0/24"),
|
||||
netip.MustParsePrefix("240.0.0.0/4"),
|
||||
netip.MustParsePrefix("2001::/23"),
|
||||
netip.MustParsePrefix("2001:db8::/32"),
|
||||
netip.MustParsePrefix("2002::/16"),
|
||||
netip.MustParsePrefix("3fff::/20"),
|
||||
}
|
||||
|
||||
func publicImageIP(ip netip.Addr) bool {
|
||||
ip = ip.Unmap()
|
||||
if !ip.IsGlobalUnicast() || ip.IsPrivate() || ip.IsLoopback() || ip.IsLinkLocalUnicast() {
|
||||
return false
|
||||
}
|
||||
// 不接受 IPv6 地址转换前缀，避免其中嵌入的 IPv4 地址访问内网。
|
||||
if ip.Is6() && !netip.MustParsePrefix("2000::/3").Contains(ip) {
|
||||
return false
|
||||
}
|
||||
for _, prefix := range blockedImageNetworks {
|
||||
if prefix.Contains(ip) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func checkImageURL(u *url.URL) error {
|
||||
if (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" || u.User != nil {
|
||||
return errImageAddress
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var remoteImageClient = &http.Client{
|
||||
Timeout: 10 * time.Second,
|
||||
Transport: &http.Transport{
|
||||
// 不使用环境代理，确保每个实际连接都经过目标地址校验。
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: 5 * time.Second,
|
||||
ControlContext: func(_ context.Context, _, address string, _ syscall.RawConn) error {
|
||||
host, _, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ip, err := netip.ParseAddr(host)
|
||||
if err != nil || !publicImageIP(ip) {
|
||||
return errImageAddress
|
||||
}
|
||||
local, err := net.InterfaceAddrs()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, addr := range local {
|
||||
prefix, err := netip.ParsePrefix(addr.String())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ip.Unmap() == prefix.Addr().Unmap() {
|
||||
return errImageAddress
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}).DialContext,
|
||||
TLSHandshakeTimeout: 5 * time.Second,
|
||||
IdleConnTimeout: 30 * time.Second,
|
||||
ForceAttemptHTTP2: true,
|
||||
},
|
||||
CheckRedirect: func(r *http.Request, via []*http.Request) error {
|
||||
if len(via) >= 5 {
|
||||
return errors.New("图片来源重定向次数过多")
|
||||
}
|
||||
r.Header.Del("Referer")
|
||||
return checkImageURL(r.URL)
|
||||
},
|
||||
}
|
||||
|
||||
// readImageDownload 按实际文件签名确定类型，不信任来源的扩展名或 Content-Type。
|
||||
func readImageDownload(body io.Reader) ([]byte, string, error) {
|
||||
data, err := io.ReadAll(io.LimitReader(body, maxAssetBytes+1))
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if int64(len(data)) > maxAssetBytes {
|
||||
return nil, "", vault.ErrTooLarge
|
||||
}
|
||||
var ext string
|
||||
switch http.DetectContentType(data) {
|
||||
case "image/png":
|
||||
ext = ".png"
|
||||
case "image/jpeg":
|
||||
ext = ".jpg"
|
||||
case "image/gif":
|
||||
ext = ".gif"
|
||||
case "image/webp":
|
||||
ext = ".webp"
|
||||
default:
|
||||
return nil, "", errImageType
|
||||
}
|
||||
return data, ext, nil
|
||||
}
|
||||
|
||||
func (s *Server) importAsset(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct{ URL string }
|
||||
if !readJSON(w, r, &body) {
|
||||
return
|
||||
}
|
||||
u, err := url.Parse(body.URL)
|
||||
if err != nil || checkImageURL(u) != nil {
|
||||
writeError(w, 400, errImageAddress.Error())
|
||||
return
|
||||
}
|
||||
note := r.URL.Query().Get("note")
|
||||
if !vault.IsNote(note) {
|
||||
writeError(w, 400, "所属笔记路径无效")
|
||||
return
|
||||
}
|
||||
if _, err := s.vault.Stat(note); err != nil {
|
||||
vaultError(w, r, err)
|
||||
return
|
||||
}
|
||||
for attempt := 0; attempt < 2; attempt++ {
|
||||
if attempt > 0 {
|
||||
timer := time.NewTimer(time.Second)
|
||||
select {
|
||||
case <-r.Context().Done():
|
||||
timer.Stop()
|
||||
return
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, u.String(), nil)
|
||||
if err != nil {
|
||||
writeError(w, 400, errImageAddress.Error())
|
||||
return
|
||||
}
|
||||
res, err := remoteImageClient.Do(req)
|
||||
if err != nil {
|
||||
var timeout net.Error
|
||||
if attempt == 0 && errors.As(err, &timeout) && timeout.Timeout() && r.Context().Err() == nil {
|
||||
continue
|
||||
}
|
||||
if errors.Is(err, errImageAddress) {
|
||||
writeError(w, 400, errImageAddress.Error())
|
||||
return
|
||||
}
|
||||
// url.Error 含完整来源 URL，其中可能有临时签名，日志只保留底层原因。
|
||||
var urlError *url.Error
|
||||
if errors.As(err, &urlError) {
|
||||
err = urlError.Err
|
||||
}
|
||||
slog.Error("下载图片失败", "err", err)
|
||||
writeError(w, 502, "下载图片失败")
|
||||
return
|
||||
}
|
||||
if res.StatusCode < 200 || res.StatusCode >= 300 {
|
||||
res.Body.Close()
|
||||
if attempt == 0 && (res.StatusCode == 429 || res.StatusCode >= 500 && res.StatusCode < 600) {
|
||||
continue
|
||||
}
|
||||
writeError(w, 502, fmt.Sprintf("图片来源返回 HTTP %d", res.StatusCode))
|
||||
return
|
||||
}
|
||||
data, ext, err := readImageDownload(res.Body)
|
||||
res.Body.Close()
|
||||
if err != nil {
|
||||
var timeout net.Error
|
||||
if attempt == 0 && errors.As(err, &timeout) && timeout.Timeout() && r.Context().Err() == nil {
|
||||
continue
|
||||
}
|
||||
if errors.Is(err, vault.ErrTooLarge) {
|
||||
vaultError(w, r, err)
|
||||
} else if errors.Is(err, errImageType) {
|
||||
writeError(w, 422, err.Error())
|
||||
} else {
|
||||
slog.Error("读取下载图片失败", "err", err)
|
||||
writeError(w, 502, "读取下载图片失败")
|
||||
}
|
||||
return
|
||||
}
|
||||
// 下载期间笔记可能被外部程序移动或删除；不能在旧路径下创建孤立资源。
|
||||
info, err := s.vault.Stat(note)
|
||||
if err != nil {
|
||||
vaultError(w, r, err)
|
||||
return
|
||||
}
|
||||
if info.IsDir() {
|
||||
writeError(w, 400, "所属路径必须是笔记文件")
|
||||
return
|
||||
}
|
||||
link, err := s.vault.SaveAsset(note, "image"+ext, bytes.NewReader(data), maxAssetBytes)
|
||||
if err != nil {
|
||||
vaultError(w, r, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, map[string]string{"link": link})
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/netip"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"rawmark/internal/vault"
|
||||
)
|
||||
|
||||
func TestImportImageAddressBoundary(t *testing.T) {
|
||||
for _, address := range []string{"127.0.0.1", "10.0.0.1", "172.16.0.1", "192.168.0.1", "169.254.169.254", "100.100.100.200", "0.1.2.3", "198.18.0.1", "224.0.0.1", "240.0.0.1", "::", "::1", "::ffff:127.0.0.1", "fc00::1", "fe80::1", "64:ff9b::7f00:1", "2002:7f00:1::", "2001::1", "2001:db8::1", "3fff::1"} {
|
||||
if publicImageIP(netip.MustParseAddr(address)) {
|
||||
t.Errorf("不应接受内网或特殊用途地址 %s", address)
|
||||
}
|
||||
}
|
||||
for _, address := range []string{"1.1.1.1", "8.8.8.8", "2606:4700:4700::1111"} {
|
||||
if !publicImageIP(netip.MustParseAddr(address)) {
|
||||
t.Errorf("应接受公网地址 %s", address)
|
||||
}
|
||||
}
|
||||
f := newFixture(t)
|
||||
f.write("n.md", "正文")
|
||||
for _, source := range []string{"file:///etc/passwd", "ftp://example.com/a.png", "https://user:[email protected]/a.png", "http://127.0.0.1/a.png", "http://[::1]/a.png", "http://localhost/a.png"} {
|
||||
body, err := json.Marshal(map[string]string{"url": source})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if code := f.status(f.admin, "POST", "/api/v1/assets/import?note=n.md", string(body), nil); code != 400 {
|
||||
t.Fatalf("应拒绝非公网图片来源，得到 %d", code)
|
||||
}
|
||||
}
|
||||
if code := f.status(newClient(), "POST", "/api/v1/assets/import?note=n.md", `{"url":"https://example.com/a.png"}`, nil); code != 401 {
|
||||
t.Fatalf("未登录不允许下载: %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportImageDownloadAndRedirects(t *testing.T) {
|
||||
png, err := base64.StdEncoding.DecodeString("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aT6sAAAAASUVORK5CYII=")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
attempts := map[string]int{}
|
||||
var ownerRoot string
|
||||
source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
attempts[r.URL.Path]++
|
||||
if r.Header.Get("Cookie") != "" || r.Header.Get("Authorization") != "" || r.Header.Get("Referer") != "" {
|
||||
t.Error("下载请求泄露了凭据或来源")
|
||||
}
|
||||
switch r.URL.Path {
|
||||
case "/private":
|
||||
http.Redirect(w, r, "http://127.0.0.1/private.png", http.StatusFound)
|
||||
case "/scheme":
|
||||
http.Redirect(w, r, "file:///etc/passwd", http.StatusFound)
|
||||
case "/redirect":
|
||||
http.Redirect(w, r, "/image", http.StatusFound)
|
||||
case "/loop":
|
||||
http.Redirect(w, r, "/loop", http.StatusFound)
|
||||
case "/retry":
|
||||
if attempts[r.URL.Path] == 1 {
|
||||
w.WriteHeader(503)
|
||||
return
|
||||
}
|
||||
w.Write(png)
|
||||
case "/failed":
|
||||
w.WriteHeader(429)
|
||||
case "/missing":
|
||||
w.WriteHeader(404)
|
||||
case "/renamed":
|
||||
if err := os.Rename(filepath.Join(ownerRoot, "moving.md"), filepath.Join(ownerRoot, "moved.md")); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
w.Write(png)
|
||||
case "/html":
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
io.WriteString(w, "<script>not an image</script>")
|
||||
default:
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.Write(png)
|
||||
}
|
||||
}))
|
||||
defer source.Close()
|
||||
original := remoteImageClient
|
||||
transport := original.Transport.(*http.Transport).Clone()
|
||||
dial := transport.DialContext
|
||||
transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
// 只把固定的测试来源映射到本地真实 HTTP 服务，其他目标仍走生产地址校验。
|
||||
if address == "images.example:80" {
|
||||
return (&net.Dialer{}).DialContext(ctx, network, source.Listener.Addr().String())
|
||||
}
|
||||
return dial(ctx, network, address)
|
||||
}
|
||||
client := *original
|
||||
client.Transport = transport
|
||||
remoteImageClient = &client
|
||||
t.Cleanup(func() { transport.CloseIdleConnections(); remoteImageClient = original })
|
||||
f := newFixture(t)
|
||||
ownerRoot = f.root
|
||||
f.write("n.md", "正文")
|
||||
for _, route := range []string{"/image", "/redirect", "/retry"} {
|
||||
body := `{"url":"http://images.example` + route + `"}`
|
||||
result := decode[map[string]string](t, f.do(f.admin, "POST", "/api/v1/assets/import?note=n.md", body, nil))
|
||||
data, err := os.ReadFile(filepath.Join(f.root, filepath.FromSlash(result["link"])))
|
||||
if err != nil || !bytes.Equal(data, png) || !strings.HasSuffix(result["link"], ".png") {
|
||||
t.Fatalf("导入内容、类型或路径不符: %v", err)
|
||||
}
|
||||
}
|
||||
for route, expected := range map[string]int{"/private": 400, "/scheme": 400, "/loop": 502, "/html": 422, "/missing": 502, "/failed": 502} {
|
||||
body := `{"url":"http://images.example` + route + `"}`
|
||||
if code := f.status(f.admin, "POST", "/api/v1/assets/import?note=n.md", body, nil); code != expected {
|
||||
t.Errorf("%s 应返回 %d，得到 %d", route, expected, code)
|
||||
}
|
||||
}
|
||||
if attempts["/retry"] != 2 || attempts["/failed"] != 2 || attempts["/missing"] != 1 || attempts["/loop"] != 5 {
|
||||
t.Fatalf("重试或重定向没有遵守上限: %v", attempts)
|
||||
}
|
||||
f.write("moving.md", "正文")
|
||||
if code := f.status(f.admin, "POST", "/api/v1/assets/import?note=moving.md", `{"url":"http://images.example/renamed"}`, nil); code != 404 {
|
||||
t.Fatalf("下载期间移动的笔记不应写入旧路径: %d", code)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(f.root, "moving.assets")); !os.IsNotExist(err) {
|
||||
t.Fatalf("不应在旧位置创建资源: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadImageDownloadSizeLimit(t *testing.T) {
|
||||
_, _, err := readImageDownload(bytes.NewReader(make([]byte, maxAssetBytes+1)))
|
||||
if !errors.Is(err, vault.ErrTooLarge) {
|
||||
t.Fatalf("超限图片应被拒绝: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -790,6 +790,111 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/assets/import": {
|
||||
"post": {
|
||||
"summary": "下载公网图片到笔记的 .assets 目录",
|
||||
"tags": [
|
||||
"图片"
|
||||
],
|
||||
"description": "仅接受公网 HTTP/HTTPS 地址，实际连接及重定向均校验；不携带登录凭据或使用环境代理。重定向链最多 5 个请求、2 次下载尝试，单次超时 10 秒；仅超时、429 和 5xx 重试。支持 PNG/JPEG/GIF/WebP，按文件签名识别，最大 50 MB。",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "note",
|
||||
"in": "query",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"requestBody": {
|
||||
"required": true,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"url"
|
||||
],
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string",
|
||||
"format": "uri"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "相对笔记所在目录的图片链接",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"link": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "来源地址或笔记路径无效，或目标不是公网",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"404": {
|
||||
"description": "所属笔记不存在",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"413": {
|
||||
"description": "图片超过 50 MB",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"422": {
|
||||
"description": "不支持的图片类型",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"502": {
|
||||
"description": "来源请求失败",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/raw/{path}": {
|
||||
"parameters": [
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user