feat(import): restore MindSift data and import Obsidian vaults
MindSift CI & Docker Release / Build, Test in Container & Push (push) Successful in 1m28s
MindSift CI & Docker Release / Build, Test in Container & Push (push) Successful in 1m28s
- Two separate paths: POST /import/mindsift (JSON backup or the Markdown archive) and POST /import/obsidian (vault ZIP into a folder named after it) - Everything is planned and validated first; ?preview=1 returns what would be created and what is skipped (with reason), nothing is written - MindSift data round-trips: folders, note structure, task lists with subtasks/due dates/priorities/notes, lists with quantity/category/done, triage thoughts from the JSON backup - Markdown to outliner: nested lists, heading sections, callouts, code, paragraphs; front matter tags become hashtags; .obsidian/attachments skipped - Limits against zip bombs and path tricks (100 MB upload, 10 MB per note, 500 MB unpacked, 20k files, no .. escapes) - Settings: two import cards with drag & drop, preview, confirm, result; touch-sized buttons; mobile download/upload verified Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
d8a16c6775
commit
1601a6a2d5
9 files changed
+1314
-1
No files matched your search
@@ -0,0 +1,199 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/sandrodach/mindsift/internal/domain"
|
||||
)
|
||||
|
||||
func newImportStore(t *testing.T) (*Store, *domain.User, *domain.User) {
|
||||
t.Helper()
|
||||
store, err := NewStore(filepath.Join(t.TempDir(), "import.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("NewStore: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { store.Close() })
|
||||
openSharing(t, store)
|
||||
anna, _ := store.CreateUser("anna@example.com", "hash", "Anna", "member")
|
||||
ben, _ := store.CreateUser("ben@example.com", "hash", "Ben", "member")
|
||||
return store, anna, ben
|
||||
}
|
||||
|
||||
// seedAnna creates one of everything the import has to bring back.
|
||||
func seedAnna(t *testing.T, store *Store, anna *domain.User) {
|
||||
t.Helper()
|
||||
blocks := []domain.DocumentBlock{
|
||||
{ID: "b1", Type: "h2", Content: "Warum", Level: 0},
|
||||
{ID: "b2", Type: "bullet", Content: "Strom sparen #homelab", Level: 1},
|
||||
{ID: "b3", Type: "todo", Content: "Backup", Checked: true, Level: 2},
|
||||
{ID: "b4", Type: "code", Content: "docker compose up", Lang: "bash", Level: 2},
|
||||
}
|
||||
if _, err := store.CreateDocument(anna.ID, "", "Umzug", "🚀", "Projekte/Server", blocks, nil, false); err != nil {
|
||||
t.Fatalf("CreateDocument: %v", err)
|
||||
}
|
||||
tl, _ := store.CreateTaskList(anna.ID, "", "", "Haushalt", "🏠", "")
|
||||
task, _ := store.CreateThoughtWithDetails(anna.ID, "Fenster putzen", "urgent", "manual", "2026-11-02", tl.ID, "todo", "Leiter im Keller")
|
||||
_, _ = store.UpdateThoughtTriage(anna.ID, task.ID, "keep", "task", "", "2026-11-02", tl.ID)
|
||||
subs := []domain.SubTask{{ID: "s1", Title: "Leiter holen", IsCompleted: true}}
|
||||
_, _ = store.UpdateThoughtDetails(anna.ID, task.ID, nil, nil, nil, nil, nil, nil, nil, &subs, nil)
|
||||
list, _ := store.CreateList(anna.ID, "Einkauf", "🛒", "shopping")
|
||||
item, _ := store.CreateListItem(list.ID, "Milch", "Kühlregal", "2L", 0, anna.ID)
|
||||
_, _ = store.ToggleListItem(item.ID, true, anna.ID, "Anna")
|
||||
}
|
||||
|
||||
func checkRestored(t *testing.T, store *Store, ben *domain.User) {
|
||||
t.Helper()
|
||||
docs, _ := store.GetDocuments(ben.ID)
|
||||
if len(docs) != 1 || docs[0].Title != "Umzug" || docs[0].FolderPath != "Projekte/Server" || docs[0].Icon != "🚀" {
|
||||
t.Fatalf("note not restored: %+v", docs)
|
||||
}
|
||||
got := []string{}
|
||||
for _, b := range docs[0].Blocks {
|
||||
got = append(got, b.Type+"@"+string(rune('0'+b.Level))+":"+b.Content)
|
||||
}
|
||||
want := []string{"h2@0:Warum", "bullet@1:Strom sparen #homelab", "todo@2:Backup", "code@2:docker compose up"}
|
||||
if strings.Join(got, "|") != strings.Join(want, "|") {
|
||||
t.Errorf("blocks differ:\n got %v\nwant %v", got, want)
|
||||
}
|
||||
lists, _ := store.GetTaskLists(ben.ID)
|
||||
tasks, _ := store.GetTasks(ben.ID, "")
|
||||
if len(lists) != 1 || lists[0].Title != "Haushalt" || len(tasks) != 1 {
|
||||
t.Fatalf("task list not restored: %+v / %+v", lists, tasks)
|
||||
}
|
||||
tk := tasks[0]
|
||||
if tk.Content != "Fenster putzen" || tk.Priority != "urgent" || tk.DueDate != "2026-11-02" || tk.Notes != "Leiter im Keller" ||
|
||||
len(tk.Subtasks) != 1 || !tk.Subtasks[0].IsCompleted || tk.TaskListID != lists[0].ID {
|
||||
t.Errorf("task not restored: %+v", tk)
|
||||
}
|
||||
shop, _ := store.GetAllListsForUser(ben.ID)
|
||||
if len(shop) != 1 || len(shop[0].Items) != 1 {
|
||||
t.Fatalf("list not restored: %+v", shop)
|
||||
}
|
||||
it := shop[0].Items[0]
|
||||
if it.Content != "Milch" || it.Quantity != "2L" || it.Category != "Kühlregal" || !it.IsCompleted {
|
||||
t.Errorf("list item not restored: %+v", it)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportMindSiftArchiveRoundTrip(t *testing.T) {
|
||||
store, anna, ben := newImportStore(t)
|
||||
seedAnna(t, store, anna)
|
||||
var buf bytes.Buffer
|
||||
if err := store.WriteMarkdownVault(anna.ID, &buf); err != nil {
|
||||
t.Fatalf("export: %v", err)
|
||||
}
|
||||
plan, err := PlanMindSiftArchiveImport(bytes.NewReader(buf.Bytes()), int64(buf.Len()))
|
||||
if err != nil {
|
||||
t.Fatalf("plan: %v", err)
|
||||
}
|
||||
if sum := plan.Summary(false); sum.Notes != 1 || sum.TaskLists != 1 || sum.Tasks != 1 || sum.Lists != 1 || sum.ListItems != 1 {
|
||||
t.Fatalf("preview counts wrong: %+v", sum)
|
||||
}
|
||||
// The preview alone writes nothing
|
||||
if docs, _ := store.GetDocuments(ben.ID); len(docs) != 0 {
|
||||
t.Fatalf("preview must not import")
|
||||
}
|
||||
if _, err := store.ApplyImport(ben.ID, plan); err != nil {
|
||||
t.Fatalf("apply: %v", err)
|
||||
}
|
||||
checkRestored(t, store, ben)
|
||||
}
|
||||
|
||||
func TestImportMindSiftJSONRoundTrip(t *testing.T) {
|
||||
store, anna, ben := newImportStore(t)
|
||||
seedAnna(t, store, anna)
|
||||
exp, _ := store.ExportUserData(anna.ID)
|
||||
data, _ := json.Marshal(exp)
|
||||
plan, err := PlanMindSiftJSONImport(data)
|
||||
if err != nil {
|
||||
t.Fatalf("plan: %v", err)
|
||||
}
|
||||
if _, err := store.ApplyImport(ben.ID, plan); err != nil {
|
||||
t.Fatalf("apply: %v", err)
|
||||
}
|
||||
checkRestored(t, store, ben)
|
||||
|
||||
if _, err := PlanMindSiftJSONImport([]byte(`{"foo":1}`)); err == nil {
|
||||
t.Errorf("foreign JSON must be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func zipOf(t *testing.T, files map[string]string) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
for name, body := range files {
|
||||
w, _ := zw.Create(name)
|
||||
_, _ = w.Write([]byte(body))
|
||||
}
|
||||
_ = zw.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func TestImportObsidianVault(t *testing.T) {
|
||||
store, _, ben := newImportStore(t)
|
||||
data := zipOf(t, map[string]string{
|
||||
"Mein Vault/Notizen/Idee.md": "---\ntags: [projekt, ideen]\naliases:\n - X\n---\n" +
|
||||
"Ein Absatz mit [[Wikilink]]\nüber zwei Zeilen.\n\n" +
|
||||
"> [!tip] Wichtig\n> Callout-Text\n\n" +
|
||||
"## Liste\n- Punkt A\n - Unterpunkt #projekt\n - [x] Erledigt\n- Punkt B\n\n" +
|
||||
"```go\nfmt.Println(1)\n```\n",
|
||||
"Mein Vault/Willkommen.md": "# Hallo\nText",
|
||||
"Mein Vault/bild.png": "PNG",
|
||||
"Mein Vault/.obsidian/app.json": "{}",
|
||||
"Mein Vault/../../boese.md": "- ausgebrochen?",
|
||||
})
|
||||
plan, err := PlanObsidianImport(bytes.NewReader(data), int64(len(data)), "Mein Vault.zip")
|
||||
if err != nil {
|
||||
t.Fatalf("plan: %v", err)
|
||||
}
|
||||
sum := plan.Summary(false)
|
||||
if sum.Notes != 3 || len(sum.Skipped) != 1 || !strings.Contains(sum.Skipped[0].Path, "bild.png") {
|
||||
t.Fatalf("unexpected preview: %+v", sum)
|
||||
}
|
||||
if _, err := store.ApplyImport(ben.ID, plan); err != nil {
|
||||
t.Fatalf("apply: %v", err)
|
||||
}
|
||||
docs, _ := store.GetDocuments(ben.ID)
|
||||
byTitle := map[string]domain.Document{}
|
||||
for _, d := range docs {
|
||||
byTitle[d.Title] = d
|
||||
if !strings.HasPrefix(d.FolderPath, "Mein Vault") {
|
||||
t.Errorf("note %q outside the vault folder: %q", d.Title, d.FolderPath)
|
||||
}
|
||||
}
|
||||
idee, ok := byTitle["Idee"]
|
||||
if !ok || idee.FolderPath != "Mein Vault/Notizen" {
|
||||
t.Fatalf("Idee not in Mein Vault/Notizen: %+v", byTitle)
|
||||
}
|
||||
got := []string{}
|
||||
for _, b := range idee.Blocks {
|
||||
got = append(got, b.Type+"@"+string(rune('0'+b.Level))+":"+b.Content)
|
||||
}
|
||||
want := []string{
|
||||
"bullet@0:Ein Absatz mit [[Wikilink]]\nüber zwei Zeilen.",
|
||||
"callout@0:Wichtig\nCallout-Text",
|
||||
"h2@0:Liste",
|
||||
"bullet@1:Punkt A",
|
||||
"bullet@2:Unterpunkt #projekt",
|
||||
"todo@2:Erledigt",
|
||||
"bullet@1:Punkt B",
|
||||
"code@1:fmt.Println(1)",
|
||||
"bullet@0:#ideen", // front matter tag not yet in the text
|
||||
}
|
||||
if strings.Join(got, "|") != strings.Join(want, "|") {
|
||||
t.Errorf("blocks differ:\n got %q\nwant %q", got, want)
|
||||
}
|
||||
if _, ok := byTitle["boese"]; !ok {
|
||||
t.Errorf("a ../ path must be imported inside the vault, not dropped silently")
|
||||
}
|
||||
|
||||
if _, err := PlanObsidianImport(bytes.NewReader([]byte("kein zip")), 8, "x.zip"); err == nil {
|
||||
t.Errorf("invalid archive must be rejected")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user