feat(media): images and recordings in backups, exports and imports (M6)
MindSift CI & Docker Release / Build, Test in Container & Push (push) Successful in 1m52s
MindSift CI & Docker Release / Build, Test in Container & Push (push) Successful in 1m52s
- Backup ZIP (download and automatic per-user server backups) carries the files of the user's own notes as media/<id><ext>; restoring creates them with new IDs and rewrites the blocks. - Markdown archive puts them under attachments/ and embeds them relative to each note (); re-importing turns them back into image/audio blocks. - Obsidian import takes images and audio a note embeds on its own (![[file.png]], ); unused or invalid files appear as skipped in the preview instead of vanishing. - Format detection and file paths moved into storage for this. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
sandrodachandClaude Opus 5.5 committed
1 parent
3f2bc96732
commit
1f63e08a73
13 files changed
+608
-82
No files matched your search
@@ -7,7 +7,10 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -23,6 +26,7 @@ const (
|
||||
maxImportFileBytes = 10 << 20 // one unpacked note
|
||||
maxImportUnpackedBytes = 500 << 20 // all unpacked notes together
|
||||
maxImportFiles = 20000
|
||||
maxImportMediaBytes = 100 << 20 // one image or recording
|
||||
)
|
||||
|
||||
// ImportPlan is everything an import would create. It is built completely (and validated) before
|
||||
@@ -35,13 +39,15 @@ type ImportPlan struct {
|
||||
Inbox []importTask `json:"-"`
|
||||
Themes []domain.UserTheme `json:"-"`
|
||||
Skipped []ImportSkip `json:"skipped"`
|
||||
// Images and recordings of the archive: "media:<id>" (own backup) or the path in the archive
|
||||
Media map[string][]byte `json:"-"`
|
||||
}
|
||||
|
||||
// Only keeps the chosen sections of the plan (see NormalizeSections).
|
||||
func (p *ImportPlan) Only(sections map[string]bool) *ImportPlan {
|
||||
out := &ImportPlan{Skipped: p.Skipped}
|
||||
if sections[SectionNotes] {
|
||||
out.Folders, out.Docs = p.Folders, p.Docs
|
||||
out.Folders, out.Docs, out.Media = p.Folders, p.Docs, p.Media
|
||||
}
|
||||
if sections[SectionTasks] {
|
||||
out.TaskLists = p.TaskLists
|
||||
@@ -79,6 +85,7 @@ type ImportSummary struct {
|
||||
|
||||
type importDoc struct {
|
||||
folder, title, icon string
|
||||
dir string // folder of the note inside the archive, for relative image links
|
||||
blocks []domain.DocumentBlock
|
||||
}
|
||||
|
||||
@@ -161,7 +168,23 @@ func readZipNotes(r io.ReaderAt, size int64, plan *ImportPlan) ([]zipNote, []str
|
||||
continue
|
||||
}
|
||||
if !strings.EqualFold(path.Ext(name), ".md") {
|
||||
plan.skip(name, "kein Markdown (Anhänge wie Bilder oder PDFs werden nicht importiert)")
|
||||
if !importMediaExt[strings.ToLower(path.Ext(name))] {
|
||||
plan.skip(name, "kein Markdown und kein Bild/Audio (z. B. PDFs werden nicht importiert)")
|
||||
continue
|
||||
}
|
||||
if f.UncompressedSize64 > maxImportMediaBytes {
|
||||
plan.skip(name, "zu groß (max. 100 MB pro Bild oder Aufnahme)")
|
||||
continue
|
||||
}
|
||||
data, err := readZipFile(f, maxImportMediaBytes)
|
||||
if err != nil {
|
||||
plan.skip(name, "nicht lesbar")
|
||||
continue
|
||||
}
|
||||
if unpacked += int64(len(data)); unpacked > maxImportUnpackedBytes {
|
||||
return nil, nil, "", errors.New("Archiv ist entpackt zu groß (max. 500 MB)")
|
||||
}
|
||||
plan.addMedia(name, data)
|
||||
continue
|
||||
}
|
||||
if f.UncompressedSize64 > maxImportFileBytes {
|
||||
@@ -272,8 +295,9 @@ func PlanObsidianImport(r io.ReaderAt, size int64, fileName string) (*ImportPlan
|
||||
if extra := missingTags(fm.tags, body); extra != "" {
|
||||
blocks = append(blocks, newBlock("bullet", extra, 0))
|
||||
}
|
||||
plan.Docs = append(plan.Docs, importDoc{folder: folder, title: title, icon: fm.icon, blocks: blocks})
|
||||
plan.Docs = append(plan.Docs, importDoc{folder: folder, title: title, icon: fm.icon, dir: path.Dir(n.path), blocks: blocks})
|
||||
}
|
||||
plan.reportUnusedMedia()
|
||||
plan.Folders = sortedKeys(folders)
|
||||
if len(plan.Docs) == 0 {
|
||||
return nil, errors.New("Im Archiv wurden keine Markdown-Notizen gefunden")
|
||||
@@ -315,9 +339,10 @@ func PlanMindSiftArchiveImport(r io.ReaderAt, size int64) (*ImportPlan, error) {
|
||||
if folder != "" {
|
||||
folders[folder] = true
|
||||
}
|
||||
plan.Docs = append(plan.Docs, importDoc{folder: folder, title: title, icon: fm.icon, blocks: MarkdownToBlocks(body)})
|
||||
plan.Docs = append(plan.Docs, importDoc{folder: folder, title: title, icon: fm.icon, dir: path.Dir(n.path), blocks: MarkdownToBlocks(body)})
|
||||
}
|
||||
}
|
||||
plan.reportUnusedMedia()
|
||||
plan.Folders = sortedKeys(folders)
|
||||
if len(plan.Docs)+len(plan.TaskLists)+len(plan.Lists) == 0 {
|
||||
return nil, errors.New("Das Archiv enthält keine MindSift-Daten (Notizen, Aufgaben oder Listen)")
|
||||
@@ -328,10 +353,12 @@ func PlanMindSiftArchiveImport(r io.ReaderAt, size int64) (*ImportPlan, error) {
|
||||
// BackupFileName is the JSON inside the backup ZIP.
|
||||
const BackupFileName = "mindsift-backup.json"
|
||||
|
||||
// WriteBackupZip streams the JSON backup compressed into a ZIP with a single file.
|
||||
func WriteBackupZip(exp *domain.FullExport, w io.Writer) error {
|
||||
// WriteBackupZip streams the JSON backup compressed into a ZIP, plus the images and recordings of the
|
||||
// user's notes as media/<id><ext> (blocks point to them as media:<id>).
|
||||
func (s *Store) WriteBackupZip(userID string, exp *domain.FullExport, w io.Writer) error {
|
||||
zw := zip.NewWriter(w)
|
||||
f, err := zw.CreateHeader(&zip.FileHeader{Name: BackupFileName, Method: zip.Deflate, Modified: parseISO(exp.ExportedAt, time.Now())})
|
||||
modified := parseISO(exp.ExportedAt, time.Now())
|
||||
f, err := zw.CreateHeader(&zip.FileHeader{Name: BackupFileName, Method: zip.Deflate, Modified: modified})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -340,9 +367,36 @@ func WriteBackupZip(exp *domain.FullExport, w io.Writer) error {
|
||||
if err := enc.Encode(exp); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.writeMediaFiles(zw, userID, func(m Media) string { return "media/" + m.ID + MediaExt(m.Mime) }); err != nil {
|
||||
return err
|
||||
}
|
||||
return zw.Close()
|
||||
}
|
||||
|
||||
// writeMediaFiles adds the user's media files to an archive (already compressed formats: stored as is).
|
||||
// Files missing on disk are left out.
|
||||
func (s *Store) writeMediaFiles(zw *zip.Writer, userID string, name func(Media) string) error {
|
||||
media, err := s.MediaOfUser(userID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, m := range media {
|
||||
src, err := os.Open(s.MediaFilePath(&m))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
dst, err := zw.CreateHeader(&zip.FileHeader{Name: name(m), Method: zip.Store, Modified: parseISO(m.CreatedAt, time.Now())})
|
||||
if err == nil {
|
||||
_, err = io.Copy(dst, src)
|
||||
}
|
||||
src.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PlanMindSiftImport picks the format of MindSift's own data: plain JSON backup, the backup ZIP
|
||||
// (one JSON inside) or the Markdown archive.
|
||||
func PlanMindSiftImport(data []byte, fileName string) (*ImportPlan, error) {
|
||||
@@ -367,6 +421,22 @@ func PlanMindSiftImport(data []byte, fileName string) (*ImportPlan, error) {
|
||||
if backup == nil {
|
||||
return PlanMindSiftArchiveImport(bytes.NewReader(data), int64(len(data)))
|
||||
}
|
||||
// Images and recordings next to the JSON: media/<id><ext>
|
||||
media := map[string][]byte{}
|
||||
var mediaBytes int64
|
||||
for _, f := range zr.File {
|
||||
name := cleanArchivePath(f.Name)
|
||||
if !strings.HasPrefix(name, "media/") || f.FileInfo().IsDir() || f.UncompressedSize64 > maxImportMediaBytes {
|
||||
continue
|
||||
}
|
||||
if mediaBytes += int64(f.UncompressedSize64); mediaBytes > maxImportUnpackedBytes {
|
||||
return nil, errors.New("Backup ist entpackt zu groß (max. 500 MB)")
|
||||
}
|
||||
if b, err := readZipFile(f, maxImportMediaBytes); err == nil {
|
||||
base := path.Base(name)
|
||||
media["media:"+strings.TrimSuffix(base, path.Ext(base))] = b
|
||||
}
|
||||
}
|
||||
if backup.UncompressedSize64 > maxImportUnpackedBytes {
|
||||
return nil, errors.New("Backup ist entpackt zu groß (max. 500 MB)")
|
||||
}
|
||||
@@ -379,7 +449,14 @@ func PlanMindSiftImport(data []byte, fileName string) (*ImportPlan, error) {
|
||||
if err != nil || len(raw) > maxImportUnpackedBytes {
|
||||
return nil, errors.New("Backup im Archiv ist nicht lesbar oder zu groß")
|
||||
}
|
||||
return PlanMindSiftJSONImport(raw)
|
||||
plan, err := PlanMindSiftJSONImport(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(media) > 0 {
|
||||
plan.Media = media
|
||||
}
|
||||
return plan, nil
|
||||
}
|
||||
|
||||
// PlanMindSiftJSONImport reads the JSON backup ("Backup herunterladen"). Notes shared with the
|
||||
@@ -461,7 +538,12 @@ func (s *Store) ApplyImport(userID string, plan *ImportPlan) (ImportSummary, err
|
||||
if icon == "" {
|
||||
icon = "📄"
|
||||
}
|
||||
if _, err := s.CreateDocument(userID, "", title, icon, d.folder, d.blocks, nil, false); err != nil {
|
||||
blocks, files := linkImportMedia(d, plan.Media, nil)
|
||||
doc, err := s.CreateDocument(userID, "", title, icon, d.folder, blocks, nil, false)
|
||||
if err != nil {
|
||||
return plan.Summary(false), fmt.Errorf("Notiz %q: %w", title, err)
|
||||
}
|
||||
if err := s.storeImportMedia(userID, doc.ID, files); err != nil {
|
||||
return plan.Summary(false), fmt.Errorf("Notiz %q: %w", title, err)
|
||||
}
|
||||
}
|
||||
@@ -902,3 +984,147 @@ func sortedKeys(m map[string]bool) []string {
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// --- Images and recordings ----------------------------------------------------------------------------
|
||||
|
||||
var importMediaExt = map[string]bool{".png": true, ".jpg": true, ".jpeg": true, ".gif": true, ".webp": true,
|
||||
".webm": true, ".ogg": true, ".oga": true, ".mp3": true, ".wav": true, ".m4a": true, ".aac": true}
|
||||
|
||||
// reportUnusedMedia lists files no note shows (or that are no valid image/recording) as skipped in the
|
||||
// preview and drops them from the plan.
|
||||
func (p *ImportPlan) reportUnusedMedia() {
|
||||
if len(p.Media) == 0 {
|
||||
return
|
||||
}
|
||||
used := map[string]bool{}
|
||||
for _, d := range p.Docs {
|
||||
linkImportMedia(d, p.Media, used)
|
||||
}
|
||||
for _, k := range sortedKeys(keySet(p.Media)) {
|
||||
if used[k] {
|
||||
continue
|
||||
}
|
||||
if _, _, ok := DetectMedia(p.Media[k], ""); !ok {
|
||||
p.skip(k, "kein gültiges Bild bzw. keine gültige Aufnahme")
|
||||
} else {
|
||||
p.skip(k, "wird in keiner Notiz angezeigt")
|
||||
}
|
||||
delete(p.Media, k)
|
||||
}
|
||||
}
|
||||
|
||||
func keySet(m map[string][]byte) map[string]bool {
|
||||
out := make(map[string]bool, len(m))
|
||||
for k := range m {
|
||||
out[k] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (p *ImportPlan) addMedia(name string, data []byte) {
|
||||
if p.Media == nil {
|
||||
p.Media = map[string][]byte{}
|
||||
}
|
||||
p.Media[name] = data
|
||||
}
|
||||
|
||||
func readZipFile(f *zip.File, limit int64) ([]byte, error) {
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rc.Close()
|
||||
data, err := io.ReadAll(io.LimitReader(rc, limit+1))
|
||||
if err != nil || int64(len(data)) > limit {
|
||||
return nil, errors.New("nicht lesbar oder zu groß")
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
type importMediaFile struct {
|
||||
id, kind, mime string
|
||||
data []byte
|
||||
}
|
||||
|
||||
var (
|
||||
mdEmbedRe = regexp.MustCompile(`^!\[([^\]]*)\]\(<?([^)>]+?)>?(?:\s+"[^"]*")?\)$`)
|
||||
wikiEmbedRe = regexp.MustCompile(`^!\[\[([^|\]]+)(?:\|([^\]]*))?\]\]$`)
|
||||
sizeOnlyRe = regexp.MustCompile(`^\d+(x\d+)?$`)
|
||||
)
|
||||
|
||||
// linkImportMedia turns blocks that only show a file of the archive into image/audio blocks with new
|
||||
// file IDs: media:<id> of an own backup, Markdown images relative to the note, Obsidian ![[file]]
|
||||
// embeds (found by name anywhere in the archive). Files without such a block are not imported.
|
||||
func linkImportMedia(d importDoc, media map[string][]byte, used map[string]bool) ([]domain.DocumentBlock, []importMediaFile) {
|
||||
if len(media) == 0 {
|
||||
return d.blocks, nil
|
||||
}
|
||||
blocks := make([]domain.DocumentBlock, len(d.blocks))
|
||||
copy(blocks, d.blocks)
|
||||
byKey := map[string]importMediaFile{} // one copy per note, even when shown twice
|
||||
var files []importMediaFile
|
||||
for i, b := range blocks {
|
||||
content := strings.TrimSpace(b.Content)
|
||||
key, caption := "", ""
|
||||
if ref, ok := ParseMediaContent(content); ok {
|
||||
key, caption = "media:"+ref.ID, ref.Caption
|
||||
} else if m := mdEmbedRe.FindStringSubmatch(content); m != nil {
|
||||
target := m[2]
|
||||
if u, err := url.PathUnescape(target); err == nil {
|
||||
target = u
|
||||
}
|
||||
key, caption = path.Clean(path.Join(d.dir, target)), m[1]
|
||||
} else if m := wikiEmbedRe.FindStringSubmatch(content); m != nil {
|
||||
name := strings.TrimSpace(m[1])
|
||||
for k := range media {
|
||||
if path.Base(k) == name || k == name {
|
||||
key = k
|
||||
break
|
||||
}
|
||||
}
|
||||
if !sizeOnlyRe.MatchString(m[2]) {
|
||||
caption = m[2]
|
||||
}
|
||||
}
|
||||
data, ok := media[key]
|
||||
if key == "" || !ok {
|
||||
continue
|
||||
}
|
||||
f, seen := byKey[key]
|
||||
if !seen {
|
||||
if used != nil {
|
||||
used[key] = true
|
||||
}
|
||||
mime, kind, ok := DetectMedia(data, "")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
f = importMediaFile{id: domain.NewID("med"), kind: kind, mime: mime, data: data}
|
||||
byKey[key] = f
|
||||
files = append(files, f)
|
||||
}
|
||||
blocks[i].Type = f.kind
|
||||
blocks[i].Content = MediaContent(f.kind, f.id, caption)
|
||||
}
|
||||
return blocks, files
|
||||
}
|
||||
|
||||
func (s *Store) storeImportMedia(userID, docID string, files []importMediaFile) error {
|
||||
if len(files) == 0 || s.mediaRoot == "" {
|
||||
return nil
|
||||
}
|
||||
for _, f := range files {
|
||||
m, err := s.createMediaWithID(f.id, docID, userID, f.kind, f.mime, int64(len(f.data)), "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dst := s.MediaFilePath(m)
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(dst, f.data, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in new issue
Block a user