From d8a16c677580290f9a0bff05867bfe917d287a90 Mon Sep 17 00:00:00 2001 From: Sandro Date: Thu, 29 Oct 2026 00:09:27 +0100 Subject: [PATCH 1/2] fix(smtp): AUTH LOGIN fallback when PLAIN is refused Servers like Postfix with Dovecot SASL or Exchange can fail PLAIN with 451 4.3.2 but accept LOGIN. MindSift now uses LOGIN directly when it is the only mechanism, and otherwise reconnects and retries with LOGIN after a refused PLAIN (net/smtp closes the connection on a failed AUTH). If both fail with 451, the error names both attempts and points to the server's SASL backend. LOGIN, like PLAIN, is never sent unencrypted to remote hosts. Co-Authored-By: Claude Opus 5.5 --- Doku/modules/tasks.md | 3 +- internal/api/mail.go | 89 +++++++++++++++++++++++++++++++++++++-- internal/api/mail_test.go | 65 ++++++++++++++++++++++++++-- 3 files changed, 149 insertions(+), 8 deletions(-) diff --git a/Doku/modules/tasks.md b/Doku/modules/tasks.md index 8e0977c..c39914c 100644 --- a/Doku/modules/tasks.md +++ b/Doku/modules/tasks.md @@ -41,7 +41,8 @@ Danach wird `alarm_sent = 1` gesetzt, sobald mindestens ein Kanal erfolgreich wa **E-Mail-Versand** ([mail.go](../../internal/api/mail.go)) führt den SMTP-Dialog Schritt für Schritt: -- Verbindung (Port 465: TLS von Anfang an), `EHLO` mit der Domain der Absenderadresse (nie `localhost`, das lehnen strenge Server ab), bei `SMTP_TLS` Pflicht-STARTTLS, `AUTH PLAIN`, `MAIL FROM`, `RCPT TO`, Daten. +- Verbindung (Port 465: TLS von Anfang an), `EHLO` mit der Domain der Absenderadresse (nie `localhost`, das lehnen strenge Server ab), bei `SMTP_TLS` Pflicht-STARTTLS, Anmeldung, `MAIL FROM`, `RCPT TO`, Daten. +- Anmeldung: `AUTH PLAIN`; bietet der Server nur `LOGIN` an, direkt `AUTH LOGIN`. Lehnt er PLAIN ab und bietet auch LOGIN an, baut MindSift die Verbindung neu auf und versucht `AUTH LOGIN` (Go schließt die Verbindung nach einem gescheiterten AUTH). Scheitern beide mit `451`, nennt die Meldung beide Versuche und verweist auf das SASL-Backend des Mailservers (z. B. Dovecot-Auth), denn dann liegt der Fehler dort. Nie unverschlüsselt an fremde Hosts. - `SMTP_FROM` darf einen Namen enthalten (`MindSift `): im Umschlag steht nur die Adresse, im Header Name und Adresse. - Die Nachricht hat `Date` und `Message-ID`, einen nach RFC 2047 kodierten Betreff (Umlaute, Emoji) und einen Quoted-Printable-Text. - `SMTP_SKIP_VERIFY` / Schalter „Zertifikatsprüfung deaktivieren“ akzeptiert selbstsignierte Zertifikate (Verbindung bleibt verschlüsselt). diff --git a/internal/api/mail.go b/internal/api/mail.go index 36e0123..d5dcb6c 100644 --- a/internal/api/mail.go +++ b/internal/api/mail.go @@ -5,6 +5,7 @@ import ( "crypto/rand" "crypto/tls" "encoding/hex" + "errors" "fmt" "mime" "mime/quotedprintable" @@ -21,6 +22,8 @@ type smtpSettings struct { Host, Port, User, Pass, From string TLS bool // 465: implicit TLS, otherwise STARTTLS (required when set) SkipVerify bool // accept self-signed / mismatching certificates + + authMech string // "" = automatic; "LOGIN" forces AUTH LOGIN (fallback after a refused PLAIN) } // smtpStepError names the protocol step at which the mail server refused, e.g. "RCPT TO". @@ -116,9 +119,89 @@ func buildMessage(from, to, subject, body string, now time.Time) []byte { return buf.Bytes() } -// sendMail runs the SMTP dialogue step by step instead of smtp.SendMail, so it can introduce itself -// properly, honour SkipVerify and report the exact step a server refuses. +// loginAuth implements AUTH LOGIN (username and password as separate Base64 challenges). Some servers, +// e.g. Postfix with Dovecot SASL or Exchange, fail on PLAIN but accept LOGIN. +type loginAuth struct{ user, pass, host string } + +func (a *loginAuth) Start(server *smtp.ServerInfo) (string, []byte, error) { + // Same rule as smtp.PlainAuth: never send credentials unencrypted to a remote host + if !server.TLS && a.host != "localhost" && a.host != "127.0.0.1" && a.host != "::1" { + return "", nil, fmt.Errorf("unverschlüsselte Verbindung – Anmeldung verweigert") + } + return "LOGIN", nil, nil +} + +func (a *loginAuth) Next(fromServer []byte, more bool) ([]byte, error) { + if !more { + return nil, nil + } + switch prompt := strings.ToLower(strings.TrimSpace(string(fromServer))); { + case strings.Contains(prompt, "username"), strings.Contains(prompt, "user"): + return []byte(a.user), nil + case strings.Contains(prompt, "password"), strings.Contains(prompt, "pass"): + return []byte(a.pass), nil + default: + return nil, fmt.Errorf("unerwartete AUTH-LOGIN-Frage %q", prompt) + } +} + +// authAttemptError records which mechanism failed and whether LOGIN would have been an alternative. +type authAttemptError struct { + mech string + loginOffered bool + err error +} + +func (e *authAttemptError) Error() string { return e.mech + ": " + e.err.Error() } +func (e *authAttemptError) Unwrap() error { return e.err } + +// authenticate uses PLAIN, or LOGIN when the server offers only that (or when cfg.authMech forces it). +// net/smtp closes the connection after a failed AUTH, so a fallback needs a new connection (sendMail). +func authenticate(client *smtp.Client, cfg smtpSettings) error { + _, mechs := client.Extension("AUTH") + offered := " " + strings.ToUpper(mechs) + " " + hasLogin := strings.Contains(offered, " LOGIN ") + mech := cfg.authMech + if mech == "" { + mech = "PLAIN" + if hasLogin && !strings.Contains(offered, " PLAIN ") { + mech = "LOGIN" + } + } + var a smtp.Auth = smtp.PlainAuth("", cfg.User, cfg.Pass, cfg.Host) + if mech == "LOGIN" { + a = &loginAuth{user: cfg.User, pass: cfg.Pass, host: cfg.Host} + } + if err := client.Auth(a); err != nil { + return &authAttemptError{mech: mech, loginOffered: hasLogin, err: err} + } + return nil +} + +// sendMail sends one mail. If AUTH PLAIN is refused and the server also offers LOGIN, it reconnects once +// and tries LOGIN; a 451 on both points to the mail server's SASL backend. func sendMail(cfg smtpSettings, to, subject, body string) error { + err := sendMailOnce(cfg, to, subject, body) + var first *authAttemptError + if cfg.authMech != "" || !errors.As(err, &first) || first.mech != "PLAIN" || !first.loginOffered { + return err + } + cfg.authMech = "LOGIN" + retryErr := sendMailOnce(cfg, to, subject, body) + var second *authAttemptError + if retryErr == nil || !errors.As(retryErr, &second) { + return retryErr + } + combined := fmt.Errorf("%v; %v", first, second) + if strings.Contains(first.err.Error(), "451") && strings.Contains(second.err.Error(), "451") { + combined = fmt.Errorf("%w (der Mailserver meldet einen internen Fehler bei der Anmeldung – bitte dessen SASL/Dovecot-Logs prüfen)", combined) + } + return &smtpStepError{"AUTH", combined} +} + +// sendMailOnce runs the SMTP dialogue step by step instead of smtp.SendMail, so it can introduce itself +// properly, honour SkipVerify and report the exact step a server refuses. +func sendMailOnce(cfg smtpSettings, to, subject, body string) error { fromAddr, fromHeader, err := splitAddress(cfg.From) if err != nil { return &smtpStepError{"Absender", err} @@ -162,7 +245,7 @@ func sendMail(cfg smtpSettings, to, subject, body string) error { } } if cfg.User != "" || cfg.Pass != "" { - if err := client.Auth(smtp.PlainAuth("", cfg.User, cfg.Pass, cfg.Host)); err != nil { + if err := authenticate(client, cfg); err != nil { return &smtpStepError{"AUTH", err} } } diff --git a/internal/api/mail_test.go b/internal/api/mail_test.go index 4be8459..6e2d321 100644 --- a/internal/api/mail_test.go +++ b/internal/api/mail_test.go @@ -8,6 +8,7 @@ import ( "crypto/tls" "crypto/x509" "crypto/x509/pkix" + "encoding/base64" "errors" "math/big" "net" @@ -21,8 +22,13 @@ import ( type fakeSMTP struct { addr string rcptCode string // e.g. "451 4.3.2 Internal server error" to refuse RCPT TO + // AUTH behaviour: mechanisms announced, and per mechanism a reply code to fail with ("" = accept) + authMechs string + authFail map[string]string mu sync.Mutex + authUsed []string // mechanisms tried, in order + authUser string // decoded user of a successful LOGIN ehlo string mailFrom string rcptTo string @@ -53,7 +59,7 @@ func startFakeSMTP(t *testing.T, rcptCode string) *fakeSMTP { t.Fatalf("listen: %v", err) } t.Cleanup(func() { _ = ln.Close() }) - f := &fakeSMTP{addr: ln.Addr().String(), rcptCode: rcptCode} + f := &fakeSMTP{addr: ln.Addr().String(), rcptCode: rcptCode, authMechs: "PLAIN", authFail: map[string]string{}} tlsCfg := &tls.Config{Certificates: []tls.Certificate{selfSignedCert(t)}} go func() { for { @@ -86,9 +92,9 @@ func (f *fakeSMTP) serve(conn net.Conn, tlsCfg *tls.Config) { f.ehlo = cmd[5:] f.mu.Unlock() if secure { - reply("250-fake\r\n250 AUTH PLAIN") + reply("250-fake\r\n250 AUTH " + f.authMechs) } else { - reply("250-fake\r\n250-STARTTLS\r\n250 AUTH PLAIN") + reply("250-fake\r\n250-STARTTLS\r\n250 AUTH " + f.authMechs) } case upper == "STARTTLS": reply("220 go ahead") @@ -99,7 +105,27 @@ func (f *fakeSMTP) serve(conn net.Conn, tlsCfg *tls.Config) { conn, secure = tlsConn, true r, w = bufio.NewReader(conn), bufio.NewWriter(conn) case strings.HasPrefix(upper, "AUTH "): - reply("235 ok") + mech := strings.Fields(upper)[1] + f.mu.Lock() + f.authUsed = append(f.authUsed, mech) + fail := f.authFail[mech] + f.mu.Unlock() + if mech == "LOGIN" { + reply("334 " + base64.StdEncoding.EncodeToString([]byte("Username:"))) + u, _ := r.ReadString('\n') + reply("334 " + base64.StdEncoding.EncodeToString([]byte("Password:"))) + _, _ = r.ReadString('\n') + if dec, err := base64.StdEncoding.DecodeString(strings.TrimSpace(u)); err == nil { + f.mu.Lock() + f.authUser = string(dec) + f.mu.Unlock() + } + } + if fail != "" { + reply(fail) + } else { + reply("235 ok") + } case strings.HasPrefix(upper, "MAIL FROM"): f.mu.Lock() f.mailFrom = strings.TrimSpace(cmd[len("MAIL FROM:"):]) @@ -213,3 +239,34 @@ func TestEHLONameNeverLocalhost(t *testing.T) { t.Errorf("fallback must not be localhost, got %q", got) } } + +func TestAuthFallsBackToLogin(t *testing.T) { + f := startFakeSMTP(t, "") + f.authMechs = "PLAIN LOGIN" + f.authFail["PLAIN"] = "451 4.3.2 Internal server error" + if err := sendMail(f.settings(true), "anna@example.com", "Test", "Hallo"); err != nil { + t.Fatalf("LOGIN fallback should succeed: %v", err) + } + f.mu.Lock() + defer f.mu.Unlock() + if strings.Join(f.authUsed, ",") != "PLAIN,LOGIN" || f.authUser != "alarm@firma.de" { + t.Fatalf("expected PLAIN then LOGIN with the user, got %v / %q", f.authUsed, f.authUser) + } +} + +func TestAuthLoginOnlyAndBothFailing(t *testing.T) { + f := startFakeSMTP(t, "") + f.authMechs = "LOGIN" + if err := sendMail(f.settings(true), "anna@example.com", "Test", "Hallo"); err != nil { + t.Fatalf("LOGIN-only server: %v", err) + } + + g := startFakeSMTP(t, "") + g.authMechs = "PLAIN LOGIN" + g.authFail["PLAIN"] = "451 4.3.2 Internal server error" + g.authFail["LOGIN"] = "451 4.3.2 Internal server error" + err := sendMail(g.settings(true), "anna@example.com", "Test", "Hallo") + if err == nil || !strings.Contains(err.Error(), "PLAIN") || !strings.Contains(err.Error(), "LOGIN") || !strings.Contains(err.Error(), "SASL") { + t.Fatalf("expected both mechanisms and the server hint in the error, got %v", err) + } +} From 1601a6a2d5b8a80f671a9384dff8b4dddbfbf7d3 Mon Sep 17 00:00:00 2001 From: Sandro Date: Thu, 29 Oct 2026 06:10:21 +0100 Subject: [PATCH 2/2] feat(import): restore MindSift data and import Obsidian vaults - Two separate paths: POST /import/mindsift (JSON backup or the Markdown archive) and POST /import/obsidian (vault ZIP into a folder named after it) - Everything is planned and validated first; ?preview=1 returns what would be created and what is skipped (with reason), nothing is written - MindSift data round-trips: folders, note structure, task lists with subtasks/due dates/priorities/notes, lists with quantity/category/done, triage thoughts from the JSON backup - Markdown to outliner: nested lists, heading sections, callouts, code, paragraphs; front matter tags become hashtags; .obsidian/attachments skipped - Limits against zip bombs and path tricks (100 MB upload, 10 MB per note, 500 MB unpacked, 20k files, no .. escapes) - Settings: two import cards with drag & drop, preview, confirm, result; touch-sized buttons; mobile download/upload verified Co-Authored-By: Claude Opus 5.5 --- Doku/api/rest.md | 1 + Doku/modules/workspace.md | 16 +- frontend/src/features/settings/ImportCard.tsx | 164 ++++ .../features/settings/UserSettingsView.tsx | 37 + frontend/src/lib/api.ts | 17 + frontend/src/lib/types.ts | 13 + internal/api/server.go | 64 ++ internal/storage/import_markdown.go | 804 ++++++++++++++++++ internal/storage/import_markdown_test.go | 199 +++++ 9 files changed, 1314 insertions(+), 1 deletion(-) create mode 100644 frontend/src/features/settings/ImportCard.tsx create mode 100644 internal/storage/import_markdown.go create mode 100644 internal/storage/import_markdown_test.go diff --git a/Doku/api/rest.md b/Doku/api/rest.md index 30b8207..30eb221 100644 --- a/Doku/api/rest.md +++ b/Doku/api/rest.md @@ -34,6 +34,7 @@ Basis: `/api/v1`. Antworten sind JSON; Fehler haben die Form `{"error": "Text"}` | GET | `/export/markdown` | Obsidian-kompatibles ZIP (`mindsift-vault-DATUM.zip`), direkt gestreamt: Notizen als `.md` mit YAML-Frontmatter in der Ordnerstruktur des Nutzers (inkl. geteilter), `Aufgaben/.md` im Format des Tasks-Plugins (`📅`, `⏫`/`🔺`), `Listen/.md` | | GET/POST | `/themes` | Eigene CSS-Themes; anlegen oder mit `id` ändern (`name`, `css`, max. 200 KB) | | DELETE | `/themes/{id}` | Eigenes Theme löschen | +| POST | `/import/mindsift`, `/import/obsidian` | Import (Multipart-Feld `file`, max. 100 MB): MindSift-JSON/-Archiv bzw. Obsidian-Vault-ZIP; `?preview=1` liefert nur die Zusammenfassung (`folders`, `notes`, `taskLists`, `tasks`, `lists`, `listItems`, `inbox`, `skipped`) | | GET/POST/DELETE | `/keys`, `/keys/{id}` | API-Keys verwalten (Klartext nur bei Erstellung) | ### Globale Suche (`GET /api/v1/search?q=`) diff --git a/Doku/modules/workspace.md b/Doku/modules/workspace.md index e72ee6a..1bebd17 100644 --- a/Doku/modules/workspace.md +++ b/Doku/modules/workspace.md @@ -61,5 +61,19 @@ Kontoeinstellungen → Private Webhooks & Export → „Markdown-Archiv herunter - Jede Notiz: YAML-Frontmatter (`title`, `icon`, `tags`, `created_at`, `updated_at`) und Markdown, das Obsidian direkt darstellt: Überschriften, Trenner, Hinweisboxen (`> [!note]`) und Codeblöcke der obersten Ebene als echte Blöcke, ihre Kinder rücken eine Ebene nach vorn; alles andere als Liste mit einem Tab pro Ebene, To-dos als `- [ ]`. - Dateinamen werden für alle Betriebssysteme bereinigt (keine `/ \ : * ? " < > |`, nie `.`/`..`); doppelte Titel bekommen „ (2)“. - Das ZIP wird beim Lesen der Daten direkt in die Antwort geschrieben, ohne temporäre Dateien. -- Ein Import aus ZIP gibt es noch nicht. + +## Import + +Kontoeinstellungen → Private Webhooks & Export → „Daten importieren“, mit **zwei getrennten Wegen** (Code: [import_markdown.go](../../internal/storage/import_markdown.go), [ImportCard.tsx](../../frontend/src/features/settings/ImportCard.tsx)): + +| Weg | Endpunkt | Dateien | Ergebnis | +|---|---|---|---| +| MindSift-Daten | `POST /import/mindsift` | JSON-Backup (`/export`) oder Markdown-Archiv (`/export/markdown`) | Notizen in ihre Ordner, Aufgabenlisten mit Unteraufgaben, Fälligkeit, Priorität und Notizen, Listen mit Menge, Kategorie und Erledigt-Status, beim JSON-Backup auch die Gedanken der Triage. Fremde, nur mit dem Exporteur geteilte Inhalte im JSON werden übersprungen. | +| Obsidian-Vault | `POST /import/obsidian` | ZIP eines Vaults | Alles in einen neuen Ordner mit dem Namen des Vaults, Unterordner bleiben. Checkboxen bleiben Checkboxen, Frontmatter-Tags werden zu `#Hashtags`, Callouts zu Hinweisboxen, Wikilinks bleiben `[[…]]`. Versteckte Ordner (`.obsidian`, `.trash`) werden ignoriert, Anhänge übersprungen. | + +- **Vorschau:** Die Oberfläche lädt die Datei zuerst mit `?preview=1` hoch. Der Server liest und prüft alles und meldet, was angelegt würde (inkl. übersprungener Dateien mit Grund), schreibt aber nichts. Erst „Jetzt importieren“ legt an. +- **Nie überschreiben:** Ein Import legt nur neu an; doppelte Titel sind möglich, Bestehendes bleibt unverändert. +- **Markdown → Notiz:** Listen behalten ihre Verschachtelung (Tabs oder Leerzeichen), Inhalte unter einer Überschrift werden darunter eingerückt (Gegenstück zum Export), Absätze werden zu Punkten, Codeblöcke zu Codeblöcken. +- **Grenzen:** Upload max. 100 MB, eine Notiz max. 10 MB, entpackt zusammen max. 500 MB, höchstens 20 000 Dateien. Pfade mit `..` oder absolute Pfade werden entschärft und bleiben im Zielordner. +- Bricht ein Import wegen eines Fehlers ab, bleibt das bis dahin Angelegte erhalten; die Meldung nennt das Element, an dem es scheiterte. diff --git a/frontend/src/features/settings/ImportCard.tsx b/frontend/src/features/settings/ImportCard.tsx new file mode 100644 index 0000000..4280fda --- /dev/null +++ b/frontend/src/features/settings/ImportCard.tsx @@ -0,0 +1,164 @@ +import React, { useRef, useState } from 'react'; +import { Upload, Check, AlertCircle, FileArchive } from 'lucide-react'; +import { ImportSummary } from '../../lib/types'; +import { api } from '../../lib/api'; + +interface ImportCardProps { + source: 'mindsift' | 'obsidian'; + title: string; + description: React.ReactNode; + accept: string; +} + +const summaryParts = (s: ImportSummary) => + [ + s.folders && `${s.folders} Ordner`, + s.notes && `${s.notes} ${s.notes === 1 ? 'Notiz' : 'Notizen'}`, + s.taskLists && `${s.taskLists} Aufgabenliste${s.taskLists === 1 ? '' : 'n'}`, + s.tasks && `${s.tasks} Aufgabe${s.tasks === 1 ? '' : 'n'}`, + s.lists && `${s.lists} Liste${s.lists === 1 ? '' : 'n'} mit ${s.listItems} Einträgen`, + s.inbox && `${s.inbox} Gedanke${s.inbox === 1 ? '' : 'n'} für die Triage`, + ].filter(Boolean) as string[]; + +// One import path: drop or pick a file, see a preview of what would be created, then import. +// Touch targets are large enough for phones; the file picker works on mobile browsers. +export const ImportCard: React.FC = ({ source, title, description, accept }) => { + const inputRef = useRef(null); + const [file, setFile] = useState(null); + const [preview, setPreview] = useState(null); + const [result, setResult] = useState(null); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + const [dragOver, setDragOver] = useState(false); + + const reset = () => { + setFile(null); + setPreview(null); + setError(null); + }; + + const pick = async (f?: File) => { + if (!f) return; + reset(); + setResult(null); + setFile(f); + setBusy(true); + try { + setPreview(await api.importData(source, f, true)); + } catch (err: any) { + setError(err.message || 'Datei konnte nicht gelesen werden'); + } finally { + setBusy(false); + } + }; + + const runImport = async () => { + if (!file) return; + setBusy(true); + setError(null); + try { + setResult(await api.importData(source, file, false)); + setFile(null); + setPreview(null); + } catch (err: any) { + setError(err.message || 'Import fehlgeschlagen'); + } finally { + setBusy(false); + } + }; + + const shown = result || preview; + + return ( +
+
+

{title}

+

{description}

+
+ +
{ + e.preventDefault(); + setDragOver(true); + }} + onDragLeave={() => setDragOver(false)} + onDrop={(e) => { + e.preventDefault(); + setDragOver(false); + pick(e.dataTransfer.files?.[0]); + }} + className={`p-4 border border-dashed rounded-card flex flex-col sm:flex-row items-center justify-center gap-3 text-center transition-colors ${ + dragOver ? 'border-accent bg-accent/10' : 'border-border' + }`} + > + Datei hierher ziehen oder + + { + pick(e.target.files?.[0]); + e.target.value = ''; + }} + /> +
+ + {busy &&

{preview ? 'Importiere…' : 'Lese Datei…'}

} + {error && ( +

+ + {error} +

+ )} + + {shown && !busy && ( +
+

+ {result ? : } + {result ? 'Import abgeschlossen' : `Vorschau: ${file?.name}`} +

+

+ {result ? 'Angelegt: ' : 'Wird angelegt: '} + {summaryParts(shown).join(', ') || 'nichts'}. +

+ {shown.skipped.length > 0 && ( +
+ {shown.skipped.length} übersprungen +
    + {shown.skipped.map((s, i) => ( +
  • + {s.path} – {s.reason} +
    • + ))} +
+
+ )} + {!result && ( +
+ + +
+ )} +
+ )} +
+ ); +}; diff --git a/frontend/src/features/settings/UserSettingsView.tsx b/frontend/src/features/settings/UserSettingsView.tsx index 41ad376..2409ef9 100644 --- a/frontend/src/features/settings/UserSettingsView.tsx +++ b/frontend/src/features/settings/UserSettingsView.tsx @@ -27,6 +27,7 @@ import themeTemplateCss from '../../themes/theme-template.css?raw'; import { UserAvatar, resizeImageToDataUrl } from '../../components/UserAvatar'; import { LegalLinks } from '../../components/LegalLinks'; import { AboutModal } from '../../components/AboutModal'; +import { ImportCard } from './ImportCard'; interface UserSettingsViewProps { currentUser: User; @@ -1273,6 +1274,42 @@ export const UserSettingsView: React.FC = ({ {exportingVault ? 'Wird erstellt…' : 'Markdown-Archiv herunterladen'} + + {/* Import: two separate paths, never guessed from the file */} +
+
+

Daten importieren

+

+ Ein Import legt alles neu an und ändert nichts Bestehendes. Vor dem Import siehst du, was angelegt wird. +

+
+
+ + Eigenes Backup zurückspielen: JSON-Backup (.json) oder Markdown-Archiv + (.zip). Notizen kommen in ihre Ordner, Aufgabenlisten mit Unteraufgaben, + Fälligkeiten und Prioritäten sowie Listen werden wiederhergestellt. + + } + /> + + Vault als .zip. Alle Notizen landen in einem eigenen Ordner mit dem Namen + des Vaults, Unterordner bleiben erhalten. Checkboxen bleiben Checkboxen in der Notiz, Frontmatter-Tags werden zu + #Hashtags. Bilder und andere Anhänge werden übersprungen. + + } + /> +
+
)} diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 1191a9d..af6da26 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -19,6 +19,7 @@ import { SystemSettings, SharingMode, UserTheme, + ImportSummary, Team, TaskList, TaskListShare, @@ -1076,6 +1077,22 @@ export const api = { return request('/api/v1/export'); }, + // Import of MindSift data (JSON backup or Markdown archive) or an Obsidian vault (ZIP). + // preview = true only reports what would be created. + async importData(source: 'mindsift' | 'obsidian', file: File, preview: boolean): Promise { + const token = getToken(); + const form = new FormData(); + form.append('file', file); + const res = await fetch(`/api/v1/import/${source}${preview ? '?preview=1' : ''}`, { + method: 'POST', + headers: token ? { Authorization: `Bearer ${token}` } : {}, + body: form, + }); + const data = await res.json().catch(() => ({})); + if (!res.ok) throw new Error(data.error || `Import fehlgeschlagen (HTTP ${res.status})`); + return data as ImportSummary; + }, + // Obsidian-compatible ZIP of the workspace; binary and possibly large, so no JSON helper and no timeout async exportMarkdownVault(): Promise { const token = getToken(); diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index a3d3322..fe48e41 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -417,6 +417,19 @@ export interface SharedList { createdAt: string; } +// Result of an import preview (applied = false) or of the import itself +export interface ImportSummary { + folders: number; + notes: number; + taskLists: number; + tasks: number; + lists: number; + listItems: number; + inbox: number; + skipped: { path: string; reason: string }[]; + applied: boolean; +} + // Custom CSS theme of the current user (Benutzereinstellungen > Darstellung) export interface UserTheme { id: string; diff --git a/internal/api/server.go b/internal/api/server.go index 1499234..739254f 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -110,6 +110,8 @@ func (s *Server) Router() http.Handler { r.Get("/api/v1/search", s.handleSearch) r.Get("/api/v1/export", s.handleExport) r.Get("/api/v1/export/markdown", s.handleExportMarkdown) + r.Post("/api/v1/import/mindsift", s.handleImportMindSift) + r.Post("/api/v1/import/obsidian", s.handleImportObsidian) r.Get("/api/v1/themes", s.handleGetThemes) r.Post("/api/v1/themes", s.handleSaveTheme) r.Delete("/api/v1/themes/{id}", s.handleDeleteTheme) @@ -2115,6 +2117,68 @@ func (s *Server) handleDeleteTheme(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, map[string]bool{"deleted": true}) } +// importUpload reads the multipart field "file" (max. storage.MaxImportUploadBytes). +func importUpload(w http.ResponseWriter, r *http.Request) ([]byte, string, error) { + r.Body = http.MaxBytesReader(w, r.Body, storage.MaxImportUploadBytes+1<<20) + file, header, err := r.FormFile("file") + if err != nil { + return nil, "", fmt.Errorf("Keine Datei erhalten oder Datei zu groß (max. 100 MB)") + } + defer file.Close() + data, err := io.ReadAll(io.LimitReader(file, storage.MaxImportUploadBytes+1)) + if err != nil || len(data) > storage.MaxImportUploadBytes { + return nil, "", fmt.Errorf("Datei zu groß (max. 100 MB)") + } + return data, header.Filename, nil +} + +// runImport answers a preview (?preview=1) with what would be created, otherwise imports. +func (s *Server) runImport(w http.ResponseWriter, r *http.Request, plan *storage.ImportPlan, err error) { + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + if r.URL.Query().Get("preview") == "1" { + writeJSON(w, http.StatusOK, plan.Summary(false)) + return + } + userID := auth.GetUserID(r.Context()) + sum, err := s.store.ApplyImport(userID, plan) + if err != nil { + slog.Error("Import abgebrochen", "user_id", userID, "error", err) + writeError(w, http.StatusInternalServerError, "Import abgebrochen: "+err.Error()) + return + } + writeJSON(w, http.StatusOK, sum) +} + +// handleImportMindSift restores MindSift's own data: the JSON backup or the Markdown archive (ZIP). +func (s *Server) handleImportMindSift(w http.ResponseWriter, r *http.Request) { + data, name, err := importUpload(w, r) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + if strings.HasSuffix(strings.ToLower(name), ".json") || (len(data) > 0 && data[0] == '{') { + plan, err := storage.PlanMindSiftJSONImport(data) + s.runImport(w, r, plan, err) + return + } + plan, err := storage.PlanMindSiftArchiveImport(bytes.NewReader(data), int64(len(data))) + s.runImport(w, r, plan, err) +} + +// handleImportObsidian imports an Obsidian vault (ZIP) into a folder named after the vault. +func (s *Server) handleImportObsidian(w http.ResponseWriter, r *http.Request) { + data, name, err := importUpload(w, r) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + plan, err := storage.PlanObsidianImport(bytes.NewReader(data), int64(len(data)), name) + s.runImport(w, r, plan, err) +} + // handleExportMarkdown streams an Obsidian-compatible ZIP of the user's workspace (no temp files). func (s *Server) handleExportMarkdown(w http.ResponseWriter, r *http.Request) { userID := auth.GetUserID(r.Context()) diff --git a/internal/storage/import_markdown.go b/internal/storage/import_markdown.go new file mode 100644 index 0000000..d84c774 --- /dev/null +++ b/internal/storage/import_markdown.go @@ -0,0 +1,804 @@ +package storage + +import ( + "archive/zip" + "encoding/json" + "errors" + "fmt" + "io" + "path" + "regexp" + "sort" + "strings" + "unicode/utf8" + + "github.com/sandrodach/mindsift/internal/domain" +) + +// Import limits: they keep a crafted archive (zip bomb, millions of files) from exhausting the server. +const ( + MaxImportUploadBytes = 100 << 20 // uploaded file + maxImportFileBytes = 10 << 20 // one unpacked note + maxImportUnpackedBytes = 500 << 20 // all unpacked notes together + maxImportFiles = 20000 +) + +// ImportPlan is everything an import would create. It is built completely (and validated) before +// anything is written, so the preview shows exactly what the import does. +type ImportPlan struct { + Folders []string `json:"-"` + Docs []importDoc `json:"-"` + TaskLists []importTaskList `json:"-"` + Lists []importList `json:"-"` + Inbox []importTask `json:"-"` + Skipped []ImportSkip `json:"skipped"` +} + +type ImportSkip struct { + Path string `json:"path"` + Reason string `json:"reason"` +} + +// ImportSummary is what the API returns for a preview and after the import. +type ImportSummary struct { + Folders int `json:"folders"` + Notes int `json:"notes"` + TaskLists int `json:"taskLists"` + Tasks int `json:"tasks"` + Lists int `json:"lists"` + ListItems int `json:"listItems"` + Inbox int `json:"inbox"` + Skipped []ImportSkip `json:"skipped"` + Applied bool `json:"applied"` +} + +type importDoc struct { + folder, title, icon string + blocks []domain.DocumentBlock +} + +type importTask struct { + content, priority, due, notes string + done bool + subtasks []domain.SubTask +} + +type importTaskList struct { + title, icon string + tasks []importTask +} + +type importList struct { + title, icon string + items []importItem +} + +type importItem struct { + content, category, quantity string + done bool +} + +func (p *ImportPlan) skip(path, reason string) { + p.Skipped = append(p.Skipped, ImportSkip{Path: path, Reason: reason}) +} + +func (p *ImportPlan) Summary(applied bool) ImportSummary { + sum := ImportSummary{Folders: len(p.Folders), Notes: len(p.Docs), Lists: len(p.Lists), Inbox: len(p.Inbox), Skipped: p.Skipped, Applied: applied} + for _, tl := range p.TaskLists { + if tl.title != "" { + sum.TaskLists++ + } + sum.Tasks += len(tl.tasks) + } + for _, l := range p.Lists { + sum.ListItems += len(l.items) + } + if sum.Skipped == nil { + sum.Skipped = []ImportSkip{} + } + return sum +} + +// --- Reading archives ------------------------------------------------------------------------------- + +type zipNote struct { + path string // inside the archive, "/"-separated, without a leading vault folder + text string +} + +// readZipNotes returns the Markdown files of an archive (other files are reported as skipped). A +// single top-level folder, as Obsidian and most zip tools create, is stripped and returned as name. +func readZipNotes(r io.ReaderAt, size int64, plan *ImportPlan) ([]zipNote, []string, string, error) { + zr, err := zip.NewReader(r, size) + if err != nil { + return nil, nil, "", errors.New("Die Datei ist kein gültiges ZIP-Archiv") + } + if len(zr.File) > maxImportFiles { + return nil, nil, "", fmt.Errorf("Zu viele Dateien im Archiv (max. %d)", maxImportFiles) + } + + names := make([]string, 0, len(zr.File)) + for _, f := range zr.File { + names = append(names, cleanArchivePath(f.Name)) + } + prefix := commonTopFolder(names) + + var notes []zipNote + var dirs []string + var unpacked int64 + for i, f := range zr.File { + name := strings.TrimPrefix(names[i], prefix) + if name == "" || hiddenPath(name) { + continue + } + if f.FileInfo().IsDir() { + dirs = append(dirs, strings.TrimSuffix(name, "/")) + continue + } + if !strings.EqualFold(path.Ext(name), ".md") { + plan.skip(name, "kein Markdown (Anhänge wie Bilder oder PDFs werden nicht importiert)") + continue + } + if f.UncompressedSize64 > maxImportFileBytes { + plan.skip(name, "zu groß (max. 10 MB pro Notiz)") + continue + } + rc, err := f.Open() + if err != nil { + plan.skip(name, "nicht lesbar") + continue + } + data, err := io.ReadAll(io.LimitReader(rc, maxImportFileBytes+1)) + rc.Close() + if err != nil || len(data) > maxImportFileBytes { + plan.skip(name, "nicht lesbar oder zu groß") + continue + } + if unpacked += int64(len(data)); unpacked > maxImportUnpackedBytes { + return nil, nil, "", errors.New("Archiv ist entpackt zu groß (max. 500 MB)") + } + if !utf8.Valid(data) { + plan.skip(name, "keine UTF-8-Textdatei") + continue + } + notes = append(notes, zipNote{path: name, text: strings.ReplaceAll(string(data), "\r\n", "\n")}) + } + sort.Slice(notes, func(i, j int) bool { return notes[i].path < notes[j].path }) + return notes, dirs, strings.TrimSuffix(prefix, "/"), nil +} + +// cleanArchivePath normalises an entry name and removes anything that could escape (.., absolute paths). +func cleanArchivePath(name string) string { + name = strings.ReplaceAll(name, "\\", "/") + parts := make([]string, 0) + for _, p := range strings.Split(name, "/") { + if p == "" || p == "." || p == ".." { + continue + } + parts = append(parts, p) + } + out := strings.Join(parts, "/") + if strings.HasSuffix(name, "/") && out != "" { + out += "/" + } + return out +} + +func commonTopFolder(names []string) string { + top := "" + for _, n := range names { + if n == "" || hiddenPath(n) { + continue + } + i := strings.Index(n, "/") + if i < 0 { + return "" // a file at the root: no common folder + } + if top == "" { + top = n[:i+1] + } else if top != n[:i+1] { + return "" + } + } + return top +} + +// hiddenPath skips Obsidian's settings, trash and OS junk. +func hiddenPath(p string) bool { + for _, part := range strings.Split(p, "/") { + if strings.HasPrefix(part, ".") || part == "__MACOSX" { + return true + } + } + return false +} + +// --- Planning --------------------------------------------------------------------------------------- + +// PlanObsidianImport reads a vault ZIP. Everything lands in a top folder named after the vault, so an +// import never mixes into existing notes. Checkboxes stay checkboxes in their note. +func PlanObsidianImport(r io.ReaderAt, size int64, fileName string) (*ImportPlan, error) { + plan := &ImportPlan{} + notes, dirs, vault, err := readZipNotes(r, size, plan) + if err != nil { + return nil, err + } + if vault == "" { + vault = strings.TrimSuffix(path.Base(fileName), path.Ext(fileName)) + } + root := segment(vault) + if root == "" { + root = "Obsidian-Import" + } + folders := map[string]bool{root: true} + for _, d := range dirs { + folders[path.Join(root, folderPathOf(d+"/x"))] = true + } + for _, n := range notes { + fm, body := splitFrontmatter(n.text) + folder := path.Join(root, folderPathOf(n.path)) + folders[folder] = true + title := fm.title + if title == "" { + title = strings.TrimSuffix(path.Base(n.path), path.Ext(n.path)) + } + blocks := MarkdownToBlocks(body) + // Front matter tags become hashtags, since MindSift reads tags from the text + if extra := missingTags(fm.tags, body); extra != "" { + blocks = append(blocks, newBlock("bullet", extra, 0)) + } + plan.Docs = append(plan.Docs, importDoc{folder: folder, title: title, icon: fm.icon, blocks: blocks}) + } + plan.Folders = sortedKeys(folders) + if len(plan.Docs) == 0 { + return nil, errors.New("Im Archiv wurden keine Markdown-Notizen gefunden") + } + return plan, nil +} + +// PlanMindSiftArchiveImport reads a ZIP created by "Markdown-Archiv herunterladen": notes go back to +// their folders, Aufgaben/ back into task lists, Listen/ back into lists. +func PlanMindSiftArchiveImport(r io.ReaderAt, size int64) (*ImportPlan, error) { + plan := &ImportPlan{} + notes, dirs, _, err := readZipNotes(r, size, plan) + if err != nil { + return nil, err + } + folders := map[string]bool{} + for _, d := range dirs { + if top := strings.SplitN(d, "/", 2)[0]; top != "Aufgaben" && top != "Listen" { + folders[folderPathOf(d+"/x")] = true + } + } + for _, n := range notes { + fm, body := splitFrontmatter(n.text) + title := fm.title + if title == "" { + title = strings.TrimSuffix(path.Base(n.path), path.Ext(n.path)) + } + switch folderPathOf(n.path) { + case "Aufgaben": + tl := importTaskList{title: title, icon: fm.icon, tasks: parseTaskFile(body)} + if title == "Ohne Liste" { + tl.title = "" + } + plan.TaskLists = append(plan.TaskLists, tl) + case "Listen": + plan.Lists = append(plan.Lists, importList{title: title, icon: fm.icon, items: parseListFile(body)}) + default: + folder := folderPathOf(n.path) + if folder != "" { + folders[folder] = true + } + plan.Docs = append(plan.Docs, importDoc{folder: folder, title: title, icon: fm.icon, blocks: MarkdownToBlocks(body)}) + } + } + plan.Folders = sortedKeys(folders) + if len(plan.Docs)+len(plan.TaskLists)+len(plan.Lists) == 0 { + return nil, errors.New("Das Archiv enthält keine MindSift-Daten (Notizen, Aufgaben oder Listen)") + } + return plan, nil +} + +// PlanMindSiftJSONImport reads the JSON backup ("Backup herunterladen"). Notes shared with the +// exporting user belong to someone else and are skipped; discarded thoughts are not restored. +func PlanMindSiftJSONImport(data []byte) (*ImportPlan, error) { + var exp domain.FullExport + if err := json.Unmarshal(data, &exp); err != nil || (exp.Version == "" && exp.ExportedAt == "") { + return nil, errors.New("Die Datei ist kein MindSift-JSON-Backup") + } + plan := &ImportPlan{} + folders := map[string]bool{} + for _, d := range exp.Documents { + if !d.IsOwner && d.Permission != "" && d.Permission != "owner" { + plan.skip(d.Title, "geteilte Notiz eines anderen Benutzers") + continue + } + if d.FolderPath != "" { + folders[d.FolderPath] = true + } + blocks := make([]domain.DocumentBlock, len(d.Blocks)) + for i, b := range d.Blocks { + b.ID, b.DocumentID, b.TaskID = domain.NewID("blk"), "", "" + blocks[i] = b + } + plan.Docs = append(plan.Docs, importDoc{folder: d.FolderPath, title: d.Title, icon: d.Icon, blocks: blocks}) + } + listIndex := map[string]int{} + for _, tl := range exp.TaskLists { + if !tl.IsOwner { + plan.skip(tl.Title, "geteilte Aufgabenliste eines anderen Benutzers") + continue + } + listIndex[tl.ID] = len(plan.TaskLists) + plan.TaskLists = append(plan.TaskLists, importTaskList{title: tl.Title, icon: tl.Icon}) + } + for _, t := range exp.Thoughts { + task := importTask{content: t.Content, priority: t.Priority, due: t.DueDate, notes: t.Notes, done: t.IsCompleted, subtasks: t.Subtasks} + switch { + case t.Status == domain.StatusInbox: + plan.Inbox = append(plan.Inbox, task) + case t.Status == domain.StatusKept || (t.Status == domain.StatusConverted && t.ConvertedType == "task"): + if i, ok := listIndex[t.TaskListID]; ok { + plan.TaskLists[i].tasks = append(plan.TaskLists[i].tasks, task) + } else { + plan.TaskLists = append(plan.TaskLists, importTaskList{tasks: []importTask{task}}) + } + } + } + for _, l := range exp.Lists { + if !l.IsOwner { + plan.skip(l.Title, "geteilte Liste eines anderen Benutzers") + continue + } + il := importList{title: l.Title, icon: l.Icon} + for _, it := range l.Items { + il.items = append(il.items, importItem{content: it.Content, category: it.Category, quantity: it.Quantity, done: it.IsCompleted}) + } + plan.Lists = append(plan.Lists, il) + } + plan.Folders = sortedKeys(folders) + return plan, nil +} + +// --- Applying --------------------------------------------------------------------------------------- + +// ApplyImport creates everything of the plan for the user. Existing data is never changed; imported +// items are added next to it. Task lists of the same name get merged into one new list per import. +func (s *Store) ApplyImport(userID string, plan *ImportPlan) (ImportSummary, error) { + for _, f := range plan.Folders { + _, _ = s.CreateFolder(userID, f) // already existing folders are fine + } + for _, d := range plan.Docs { + title := strings.TrimSpace(d.title) + if title == "" { + title = "Unbenannte Notiz" + } + icon := d.icon + if icon == "" { + icon = "📄" + } + if _, err := s.CreateDocument(userID, "", title, icon, d.folder, d.blocks, nil, false); err != nil { + return plan.Summary(false), fmt.Errorf("Notiz %q: %w", title, err) + } + } + for _, tl := range plan.TaskLists { + listID := "" // tasks without a list + if tl.title != "" { + icon := tl.icon + if icon == "" { + icon = "📋" + } + created, err := s.CreateTaskList(userID, "", "", tl.title, icon, "") + if err != nil { + return plan.Summary(false), fmt.Errorf("Aufgabenliste %q: %w", tl.title, err) + } + listID = created.ID + } + for _, t := range tl.tasks { + if err := s.importTaskInto(userID, listID, t, true); err != nil { + return plan.Summary(false), err + } + } + } + for _, t := range plan.Inbox { + if err := s.importTaskInto(userID, "", t, false); err != nil { + return plan.Summary(false), err + } + } + for _, l := range plan.Lists { + icon := l.icon + if icon == "" { + icon = "🛒" + } + created, err := s.CreateList(userID, l.title, icon, "shopping") + if err != nil { + return plan.Summary(false), fmt.Errorf("Liste %q: %w", l.title, err) + } + for i, it := range l.items { + item, err := s.CreateListItem(created.ID, it.content, it.category, it.quantity, i, userID) + if err != nil { + continue // empty lines and the like + } + if it.done { + _, _ = s.ToggleListItem(item.ID, true, userID, "") + } + } + } + return plan.Summary(true), nil +} + +func (s *Store) importTaskInto(userID, listID string, t importTask, asTask bool) error { + content := strings.TrimSpace(t.content) + if content == "" { + return nil + } + priority := t.priority + if priority != "low" && priority != "high" && priority != "urgent" { + priority = "normal" + } + th, err := s.CreateThoughtWithDetails(userID, content, priority, "import", t.due, listID, "todo", t.notes) + if err != nil { + return fmt.Errorf("Aufgabe %q: %w", content, err) + } + if !asTask { + return nil + } + if _, err := s.UpdateThoughtTriage(userID, th.ID, "keep", "task", "", t.due, listID); err != nil { + return fmt.Errorf("Aufgabe %q: %w", content, err) + } + if t.done || len(t.subtasks) > 0 { + var done *bool + if t.done { + done = &t.done + } + var subs *[]domain.SubTask + if len(t.subtasks) > 0 { + subs = &t.subtasks + } + _, _ = s.UpdateThoughtDetails(userID, th.ID, nil, done, nil, nil, nil, nil, nil, subs, nil) + } + return nil +} + +// --- Markdown parsing ------------------------------------------------------------------------------- + +type frontmatter struct { + title, icon string + tags []string +} + +// splitFrontmatter reads the few keys MindSift uses (title, icon, tags/tag) from YAML front matter. +// It is deliberately tolerant: unknown keys and nested structures are ignored. +func splitFrontmatter(text string) (frontmatter, string) { + var fm frontmatter + if !strings.HasPrefix(text, "---\n") { + return fm, text + } + end := strings.Index(text[4:], "\n---") + if end < 0 { + return fm, text + } + head := text[4 : 4+end] + body := strings.TrimLeft(text[4+end+4:], "\n") + key := "" + for _, line := range strings.Split(head, "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "- ") && (key == "tags" || key == "tag") { + fm.tags = append(fm.tags, yamlScalar(trimmed[2:])) + continue + } + k, v, ok := strings.Cut(line, ":") + if !ok || strings.HasPrefix(line, " ") { + continue + } + key = strings.TrimSpace(strings.ToLower(k)) + v = strings.TrimSpace(v) + switch key { + case "title": + fm.title = yamlScalar(v) + case "icon": + fm.icon = yamlScalar(v) + case "tags", "tag": + v = strings.Trim(v, "[]") + for _, t := range strings.FieldsFunc(v, func(r rune) bool { return r == ',' || r == ' ' }) { + if t = yamlScalar(t); t != "" { + fm.tags = append(fm.tags, t) + } + } + } + } + return fm, body +} + +func yamlScalar(v string) string { + v = strings.TrimSpace(v) + if strings.HasPrefix(v, `"`) { + var out string + if json.Unmarshal([]byte(v), &out) == nil { + return out + } + } + return strings.Trim(v, `"'`) +} + +func missingTags(tags []string, body string) string { + lower := strings.ToLower(body) + var out []string + for _, t := range tags { + t = strings.TrimPrefix(strings.TrimSpace(t), "#") + if t == "" || strings.Contains(lower, "#"+strings.ToLower(t)) { + continue + } + out = append(out, "#"+strings.ReplaceAll(t, " ", "_")) + } + return strings.Join(out, " ") +} + +var ( + listItemRe = regexp.MustCompile(`^([ \t]*)(?:[-*+]|\d+[.)])(?:[ \t]+(.*))?$`) + checkboxRe = regexp.MustCompile(`^\[([ xX])\][ \t]*(.*)$`) + headingRe = regexp.MustCompile(`^(#{1,6})[ \t]+(.*)$`) + fenceRe = regexp.MustCompile("^([ \\t]*)(```|~~~)(.*)$") + calloutRe = regexp.MustCompile(`^\[![\w-]+\][+-]?[ \t]*(.*)$`) +) + +func newBlock(typ, content string, level int) domain.DocumentBlock { + return domain.DocumentBlock{ID: domain.NewID("blk"), Type: typ, Content: content, Level: level} +} + +// indentWidth counts a tab as 4 columns. +func indentWidth(s string) int { + w := 0 + for _, r := range s { + if r == '\t' { + w += 4 + } else { + w++ + } + } + return w +} + +// MarkdownToBlocks turns Markdown into outliner blocks. Lists keep their nesting, headings start a +// section whose following content is nested below them (the reverse of the export), callouts and +// quotes become callout blocks, fenced code becomes a code block, plain paragraphs become bullets. +func MarkdownToBlocks(md string) []domain.DocumentBlock { + lines := strings.Split(md, "\n") + var blocks []domain.DocumentBlock + base := 0 // 1 inside a heading section + var indents []int // indent widths of the open list levels + inList := false // the previous line belonged to a list item (for continuation lines) + listLevel := func(width int) int { + for len(indents) > 0 && indents[len(indents)-1] > width { + indents = indents[:len(indents)-1] + } + if len(indents) == 0 || indents[len(indents)-1] < width { + indents = append(indents, width) + } + return base + len(indents) - 1 + } + + for i := 0; i < len(lines); i++ { + line := lines[i] + trimmed := strings.TrimSpace(line) + + if trimmed == "" { + inList = false + continue + } + + // Fenced code (also inside lists) + if m := fenceRe.FindStringSubmatch(line); m != nil { + fence, lang, prefix := m[2], strings.TrimSpace(m[3]), m[1] + var code []string + for i++; i < len(lines); i++ { + l := lines[i] + if strings.HasPrefix(strings.TrimSpace(l), fence) { + break + } + code = append(code, strings.TrimPrefix(l, prefix)) + } + level := base + if prefix != "" && len(indents) > 0 { + level = listLevel(indentWidth(prefix)) + } + b := newBlock("code", strings.Join(code, "\n"), level) + b.Lang = lang + blocks = append(blocks, b) + inList = false + continue + } + + if m := headingRe.FindStringSubmatch(line); m != nil && !strings.HasPrefix(line, " ") && !strings.HasPrefix(line, "\t") { + typ := map[int]string{1: "h1", 2: "h2"}[len(m[1])] + if typ == "" { + typ = "h3" + } + blocks = append(blocks, newBlock(typ, strings.TrimSpace(m[2]), 0)) + base, indents, inList = 1, nil, false + continue + } + + if trimmed == "---" || trimmed == "***" || trimmed == "___" { + if !strings.HasPrefix(line, " ") && !strings.HasPrefix(line, "\t") { + blocks = append(blocks, newBlock("divider", "", 0)) + base, indents, inList = 0, nil, false + continue + } + } + + // Quotes and Obsidian callouts (> [!note] …): one callout block + if strings.HasPrefix(trimmed, ">") && !strings.HasPrefix(line, " ") && !strings.HasPrefix(line, "\t") { + var parts []string + for ; i < len(lines) && strings.HasPrefix(strings.TrimSpace(lines[i]), ">"); i++ { + q := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(lines[i]), ">")) + if m := calloutRe.FindStringSubmatch(q); m != nil { + if q = strings.TrimSpace(m[1]); q == "" { + continue + } + } + parts = append(parts, q) + } + i-- + blocks = append(blocks, newBlock("callout", strings.Join(parts, "\n"), base)) + indents, inList = nil, false + continue + } + + if m := listItemRe.FindStringSubmatch(line); m != nil { + level := listLevel(indentWidth(m[1])) + content := m[2] + typ := "bullet" + checked := false + if c := checkboxRe.FindStringSubmatch(content); c != nil { + typ, checked, content = "todo", c[1] != " ", c[2] + } else if h := headingRe.FindStringSubmatch(content); h != nil { + typ = map[int]string{1: "h1", 2: "h2"}[len(h[1])] + if typ == "" { + typ = "h3" + } + content = strings.TrimSpace(h[2]) + } else if strings.TrimSpace(content) == "---" { + typ, content = "divider", "" + } else if strings.HasPrefix(content, "> ") { + typ, content = "callout", strings.TrimPrefix(content, "> ") + } + b := newBlock(typ, content, level) + b.Checked = checked + blocks = append(blocks, b) + inList = true + continue + } + + // Indented text right below a list item continues it (soft line break) + if inList && (strings.HasPrefix(line, " ") || strings.HasPrefix(line, "\t")) && len(blocks) > 0 { + last := &blocks[len(blocks)-1] + last.Content += "\n" + strings.TrimPrefix(trimmed, "> ") + continue + } + + // Plain paragraph (consecutive lines form one block) + var para []string + for ; i < len(lines); i++ { + l := strings.TrimSpace(lines[i]) + if l == "" || headingRe.MatchString(lines[i]) || listItemRe.MatchString(lines[i]) || fenceRe.MatchString(lines[i]) || strings.HasPrefix(l, ">") { + break + } + para = append(para, l) + } + i-- + blocks = append(blocks, newBlock("bullet", strings.Join(para, "\n"), base)) + indents, inList = nil, false + } + return blocks +} + +var ( + dueRe = regexp.MustCompile(`[ \t]*📅[ \t]*(\d{4}-\d{2}-\d{2})`) + priorityRe = regexp.MustCompile(`[ \t]*(🔺|⏫|🔼|🔽|⏬)`) +) + +// parseTaskFile reads a task list in Obsidian Tasks format as written by the export. +func parseTaskFile(body string) []importTask { + var tasks []importTask + for _, line := range strings.Split(body, "\n") { + if strings.TrimSpace(line) == "" { + continue + } + indented := strings.HasPrefix(line, "\t") || strings.HasPrefix(line, " ") + trimmed := strings.TrimSpace(line) + if indented && len(tasks) > 0 { + last := &tasks[len(tasks)-1] + if strings.HasPrefix(trimmed, ">") { + note := strings.TrimSpace(strings.TrimPrefix(trimmed, ">")) + if last.notes != "" { + last.notes += "\n" + } + last.notes += note + } else if m := listItemRe.FindStringSubmatch(trimmed); m != nil { + if c := checkboxRe.FindStringSubmatch(m[2]); c != nil { + last.subtasks = append(last.subtasks, domain.SubTask{ID: domain.NewID("st"), Title: c[2], IsCompleted: c[1] != " ", CreatedAt: domain.NowISO()}) + } + } + continue + } + m := listItemRe.FindStringSubmatch(trimmed) + if m == nil { + continue + } + t := importTask{content: m[2], priority: "normal"} + if c := checkboxRe.FindStringSubmatch(m[2]); c != nil { + t.done, t.content = c[1] != " ", c[2] + } + if d := dueRe.FindStringSubmatch(t.content); d != nil { + t.due = d[1] + t.content = dueRe.ReplaceAllString(t.content, "") + } + if p := priorityRe.FindStringSubmatch(t.content); p != nil { + t.priority = map[string]string{"🔺": "urgent", "⏫": "high", "🔼": "high", "🔽": "low", "⏬": "low"}[p[1]] + t.content = priorityRe.ReplaceAllString(t.content, "") + } + t.content = strings.TrimSpace(t.content) + tasks = append(tasks, t) + } + return tasks +} + +var quantityRe = regexp.MustCompile(`[ \t]*\(([^()]+)\)[ \t]*$`) + +// parseListFile reads list entries "- [ ] Milch (2L) #Kühlregal". +func parseListFile(body string) []importItem { + var items []importItem + for _, line := range strings.Split(body, "\n") { + m := listItemRe.FindStringSubmatch(strings.TrimSpace(line)) + if m == nil || strings.TrimSpace(m[2]) == "" { + continue + } + it := importItem{content: m[2]} + if c := checkboxRe.FindStringSubmatch(m[2]); c != nil { + it.done, it.content = c[1] != " ", c[2] + } + // A trailing #Category is turned into the category by CreateListItem itself + if q := quantityRe.FindStringSubmatch(stripTrailingTag(it.content)); q != nil { + it.quantity = q[1] + it.content = strings.Replace(it.content, q[0], "", 1) + } + it.content = strings.TrimSpace(it.content) + items = append(items, it) + } + return items +} + +func stripTrailingTag(s string) string { + if i := strings.LastIndex(s, " #"); i >= 0 && !strings.Contains(s[i+2:], " ") { + return s[:i] + } + return s +} + +// folderPathOf returns the folder of a "/"-separated archive path with every part made safe. +func folderPathOf(p string) string { + dir := path.Dir(p) + if dir == "." || dir == "/" { + return "" + } + parts := make([]string, 0) + for _, part := range strings.Split(dir, "/") { + if seg := segment(part); seg != "" { + parts = append(parts, seg) + } + } + return strings.Join(parts, "/") +} + +func sortedKeys(m map[string]bool) []string { + out := make([]string, 0, len(m)) + for k := range m { + if k != "" { + out = append(out, k) + } + } + sort.Strings(out) + return out +} diff --git a/internal/storage/import_markdown_test.go b/internal/storage/import_markdown_test.go new file mode 100644 index 0000000..4042f66 --- /dev/null +++ b/internal/storage/import_markdown_test.go @@ -0,0 +1,199 @@ +package storage + +import ( + "archive/zip" + "bytes" + "encoding/json" + "path/filepath" + "strings" + "testing" + + "github.com/sandrodach/mindsift/internal/domain" +) + +func newImportStore(t *testing.T) (*Store, *domain.User, *domain.User) { + t.Helper() + store, err := NewStore(filepath.Join(t.TempDir(), "import.db")) + if err != nil { + t.Fatalf("NewStore: %v", err) + } + t.Cleanup(func() { store.Close() }) + openSharing(t, store) + anna, _ := store.CreateUser("anna@example.com", "hash", "Anna", "member") + ben, _ := store.CreateUser("ben@example.com", "hash", "Ben", "member") + return store, anna, ben +} + +// seedAnna creates one of everything the import has to bring back. +func seedAnna(t *testing.T, store *Store, anna *domain.User) { + t.Helper() + blocks := []domain.DocumentBlock{ + {ID: "b1", Type: "h2", Content: "Warum", Level: 0}, + {ID: "b2", Type: "bullet", Content: "Strom sparen #homelab", Level: 1}, + {ID: "b3", Type: "todo", Content: "Backup", Checked: true, Level: 2}, + {ID: "b4", Type: "code", Content: "docker compose up", Lang: "bash", Level: 2}, + } + if _, err := store.CreateDocument(anna.ID, "", "Umzug", "🚀", "Projekte/Server", blocks, nil, false); err != nil { + t.Fatalf("CreateDocument: %v", err) + } + tl, _ := store.CreateTaskList(anna.ID, "", "", "Haushalt", "🏠", "") + task, _ := store.CreateThoughtWithDetails(anna.ID, "Fenster putzen", "urgent", "manual", "2026-11-02", tl.ID, "todo", "Leiter im Keller") + _, _ = store.UpdateThoughtTriage(anna.ID, task.ID, "keep", "task", "", "2026-11-02", tl.ID) + subs := []domain.SubTask{{ID: "s1", Title: "Leiter holen", IsCompleted: true}} + _, _ = store.UpdateThoughtDetails(anna.ID, task.ID, nil, nil, nil, nil, nil, nil, nil, &subs, nil) + list, _ := store.CreateList(anna.ID, "Einkauf", "🛒", "shopping") + item, _ := store.CreateListItem(list.ID, "Milch", "Kühlregal", "2L", 0, anna.ID) + _, _ = store.ToggleListItem(item.ID, true, anna.ID, "Anna") +} + +func checkRestored(t *testing.T, store *Store, ben *domain.User) { + t.Helper() + docs, _ := store.GetDocuments(ben.ID) + if len(docs) != 1 || docs[0].Title != "Umzug" || docs[0].FolderPath != "Projekte/Server" || docs[0].Icon != "🚀" { + t.Fatalf("note not restored: %+v", docs) + } + got := []string{} + for _, b := range docs[0].Blocks { + got = append(got, b.Type+"@"+string(rune('0'+b.Level))+":"+b.Content) + } + want := []string{"h2@0:Warum", "bullet@1:Strom sparen #homelab", "todo@2:Backup", "code@2:docker compose up"} + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Errorf("blocks differ:\n got %v\nwant %v", got, want) + } + lists, _ := store.GetTaskLists(ben.ID) + tasks, _ := store.GetTasks(ben.ID, "") + if len(lists) != 1 || lists[0].Title != "Haushalt" || len(tasks) != 1 { + t.Fatalf("task list not restored: %+v / %+v", lists, tasks) + } + tk := tasks[0] + if tk.Content != "Fenster putzen" || tk.Priority != "urgent" || tk.DueDate != "2026-11-02" || tk.Notes != "Leiter im Keller" || + len(tk.Subtasks) != 1 || !tk.Subtasks[0].IsCompleted || tk.TaskListID != lists[0].ID { + t.Errorf("task not restored: %+v", tk) + } + shop, _ := store.GetAllListsForUser(ben.ID) + if len(shop) != 1 || len(shop[0].Items) != 1 { + t.Fatalf("list not restored: %+v", shop) + } + it := shop[0].Items[0] + if it.Content != "Milch" || it.Quantity != "2L" || it.Category != "Kühlregal" || !it.IsCompleted { + t.Errorf("list item not restored: %+v", it) + } +} + +func TestImportMindSiftArchiveRoundTrip(t *testing.T) { + store, anna, ben := newImportStore(t) + seedAnna(t, store, anna) + var buf bytes.Buffer + if err := store.WriteMarkdownVault(anna.ID, &buf); err != nil { + t.Fatalf("export: %v", err) + } + plan, err := PlanMindSiftArchiveImport(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + if err != nil { + t.Fatalf("plan: %v", err) + } + if sum := plan.Summary(false); sum.Notes != 1 || sum.TaskLists != 1 || sum.Tasks != 1 || sum.Lists != 1 || sum.ListItems != 1 { + t.Fatalf("preview counts wrong: %+v", sum) + } + // The preview alone writes nothing + if docs, _ := store.GetDocuments(ben.ID); len(docs) != 0 { + t.Fatalf("preview must not import") + } + if _, err := store.ApplyImport(ben.ID, plan); err != nil { + t.Fatalf("apply: %v", err) + } + checkRestored(t, store, ben) +} + +func TestImportMindSiftJSONRoundTrip(t *testing.T) { + store, anna, ben := newImportStore(t) + seedAnna(t, store, anna) + exp, _ := store.ExportUserData(anna.ID) + data, _ := json.Marshal(exp) + plan, err := PlanMindSiftJSONImport(data) + if err != nil { + t.Fatalf("plan: %v", err) + } + if _, err := store.ApplyImport(ben.ID, plan); err != nil { + t.Fatalf("apply: %v", err) + } + checkRestored(t, store, ben) + + if _, err := PlanMindSiftJSONImport([]byte(`{"foo":1}`)); err == nil { + t.Errorf("foreign JSON must be rejected") + } +} + +func zipOf(t *testing.T, files map[string]string) []byte { + t.Helper() + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for name, body := range files { + w, _ := zw.Create(name) + _, _ = w.Write([]byte(body)) + } + _ = zw.Close() + return buf.Bytes() +} + +func TestImportObsidianVault(t *testing.T) { + store, _, ben := newImportStore(t) + data := zipOf(t, map[string]string{ + "Mein Vault/Notizen/Idee.md": "---\ntags: [projekt, ideen]\naliases:\n - X\n---\n" + + "Ein Absatz mit [[Wikilink]]\nüber zwei Zeilen.\n\n" + + "> [!tip] Wichtig\n> Callout-Text\n\n" + + "## Liste\n- Punkt A\n - Unterpunkt #projekt\n - [x] Erledigt\n- Punkt B\n\n" + + "```go\nfmt.Println(1)\n```\n", + "Mein Vault/Willkommen.md": "# Hallo\nText", + "Mein Vault/bild.png": "PNG", + "Mein Vault/.obsidian/app.json": "{}", + "Mein Vault/../../boese.md": "- ausgebrochen?", + }) + plan, err := PlanObsidianImport(bytes.NewReader(data), int64(len(data)), "Mein Vault.zip") + if err != nil { + t.Fatalf("plan: %v", err) + } + sum := plan.Summary(false) + if sum.Notes != 3 || len(sum.Skipped) != 1 || !strings.Contains(sum.Skipped[0].Path, "bild.png") { + t.Fatalf("unexpected preview: %+v", sum) + } + if _, err := store.ApplyImport(ben.ID, plan); err != nil { + t.Fatalf("apply: %v", err) + } + docs, _ := store.GetDocuments(ben.ID) + byTitle := map[string]domain.Document{} + for _, d := range docs { + byTitle[d.Title] = d + if !strings.HasPrefix(d.FolderPath, "Mein Vault") { + t.Errorf("note %q outside the vault folder: %q", d.Title, d.FolderPath) + } + } + idee, ok := byTitle["Idee"] + if !ok || idee.FolderPath != "Mein Vault/Notizen" { + t.Fatalf("Idee not in Mein Vault/Notizen: %+v", byTitle) + } + got := []string{} + for _, b := range idee.Blocks { + got = append(got, b.Type+"@"+string(rune('0'+b.Level))+":"+b.Content) + } + want := []string{ + "bullet@0:Ein Absatz mit [[Wikilink]]\nüber zwei Zeilen.", + "callout@0:Wichtig\nCallout-Text", + "h2@0:Liste", + "bullet@1:Punkt A", + "bullet@2:Unterpunkt #projekt", + "todo@2:Erledigt", + "bullet@1:Punkt B", + "code@1:fmt.Println(1)", + "bullet@0:#ideen", // front matter tag not yet in the text + } + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Errorf("blocks differ:\n got %q\nwant %q", got, want) + } + if _, ok := byTitle["boese"]; !ok { + t.Errorf("a ../ path must be imported inside the vault, not dropped silently") + } + + if _, err := PlanObsidianImport(bytes.NewReader([]byte("kein zip")), 8, "x.zip"); err == nil { + t.Errorf("invalid archive must be rejected") + } +}