feat(import): restore MindSift data and import Obsidian vaults
MindSift CI & Docker Release / Build, Test in Container & Push (push) Successful in 1m28s
MindSift CI & Docker Release / Build, Test in Container & Push (push) Successful in 1m28s
- Two separate paths: POST /import/mindsift (JSON backup or the Markdown archive) and POST /import/obsidian (vault ZIP into a folder named after it) - Everything is planned and validated first; ?preview=1 returns what would be created and what is skipped (with reason), nothing is written - MindSift data round-trips: folders, note structure, task lists with subtasks/due dates/priorities/notes, lists with quantity/category/done, triage thoughts from the JSON backup - Markdown to outliner: nested lists, heading sections, callouts, code, paragraphs; front matter tags become hashtags; .obsidian/attachments skipped - Limits against zip bombs and path tricks (100 MB upload, 10 MB per note, 500 MB unpacked, 20k files, no .. escapes) - Settings: two import cards with drag & drop, preview, confirm, result; touch-sized buttons; mobile download/upload verified Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
sandrodachandClaude Opus 5.5 committed
1 parent
d8a16c6775
commit
1601a6a2d5
9 files changed
+1314
-1
No files matched your search
@@ -0,0 +1,804 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/sandrodach/mindsift/internal/domain"
|
||||
)
|
||||
|
||||
// Import limits: they keep a crafted archive (zip bomb, millions of files) from exhausting the server.
|
||||
const (
|
||||
MaxImportUploadBytes = 100 << 20 // uploaded file
|
||||
maxImportFileBytes = 10 << 20 // one unpacked note
|
||||
maxImportUnpackedBytes = 500 << 20 // all unpacked notes together
|
||||
maxImportFiles = 20000
|
||||
)
|
||||
|
||||
// ImportPlan is everything an import would create. It is built completely (and validated) before
|
||||
// anything is written, so the preview shows exactly what the import does.
|
||||
type ImportPlan struct {
|
||||
Folders []string `json:"-"`
|
||||
Docs []importDoc `json:"-"`
|
||||
TaskLists []importTaskList `json:"-"`
|
||||
Lists []importList `json:"-"`
|
||||
Inbox []importTask `json:"-"`
|
||||
Skipped []ImportSkip `json:"skipped"`
|
||||
}
|
||||
|
||||
type ImportSkip struct {
|
||||
Path string `json:"path"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
// ImportSummary is what the API returns for a preview and after the import.
|
||||
type ImportSummary struct {
|
||||
Folders int `json:"folders"`
|
||||
Notes int `json:"notes"`
|
||||
TaskLists int `json:"taskLists"`
|
||||
Tasks int `json:"tasks"`
|
||||
Lists int `json:"lists"`
|
||||
ListItems int `json:"listItems"`
|
||||
Inbox int `json:"inbox"`
|
||||
Skipped []ImportSkip `json:"skipped"`
|
||||
Applied bool `json:"applied"`
|
||||
}
|
||||
|
||||
type importDoc struct {
|
||||
folder, title, icon string
|
||||
blocks []domain.DocumentBlock
|
||||
}
|
||||
|
||||
type importTask struct {
|
||||
content, priority, due, notes string
|
||||
done bool
|
||||
subtasks []domain.SubTask
|
||||
}
|
||||
|
||||
type importTaskList struct {
|
||||
title, icon string
|
||||
tasks []importTask
|
||||
}
|
||||
|
||||
type importList struct {
|
||||
title, icon string
|
||||
items []importItem
|
||||
}
|
||||
|
||||
type importItem struct {
|
||||
content, category, quantity string
|
||||
done bool
|
||||
}
|
||||
|
||||
func (p *ImportPlan) skip(path, reason string) {
|
||||
p.Skipped = append(p.Skipped, ImportSkip{Path: path, Reason: reason})
|
||||
}
|
||||
|
||||
func (p *ImportPlan) Summary(applied bool) ImportSummary {
|
||||
sum := ImportSummary{Folders: len(p.Folders), Notes: len(p.Docs), Lists: len(p.Lists), Inbox: len(p.Inbox), Skipped: p.Skipped, Applied: applied}
|
||||
for _, tl := range p.TaskLists {
|
||||
if tl.title != "" {
|
||||
sum.TaskLists++
|
||||
}
|
||||
sum.Tasks += len(tl.tasks)
|
||||
}
|
||||
for _, l := range p.Lists {
|
||||
sum.ListItems += len(l.items)
|
||||
}
|
||||
if sum.Skipped == nil {
|
||||
sum.Skipped = []ImportSkip{}
|
||||
}
|
||||
return sum
|
||||
}
|
||||
|
||||
// --- Reading archives -------------------------------------------------------------------------------
|
||||
|
||||
type zipNote struct {
|
||||
path string // inside the archive, "/"-separated, without a leading vault folder
|
||||
text string
|
||||
}
|
||||
|
||||
// readZipNotes returns the Markdown files of an archive (other files are reported as skipped). A
|
||||
// single top-level folder, as Obsidian and most zip tools create, is stripped and returned as name.
|
||||
func readZipNotes(r io.ReaderAt, size int64, plan *ImportPlan) ([]zipNote, []string, string, error) {
|
||||
zr, err := zip.NewReader(r, size)
|
||||
if err != nil {
|
||||
return nil, nil, "", errors.New("Die Datei ist kein gültiges ZIP-Archiv")
|
||||
}
|
||||
if len(zr.File) > maxImportFiles {
|
||||
return nil, nil, "", fmt.Errorf("Zu viele Dateien im Archiv (max. %d)", maxImportFiles)
|
||||
}
|
||||
|
||||
names := make([]string, 0, len(zr.File))
|
||||
for _, f := range zr.File {
|
||||
names = append(names, cleanArchivePath(f.Name))
|
||||
}
|
||||
prefix := commonTopFolder(names)
|
||||
|
||||
var notes []zipNote
|
||||
var dirs []string
|
||||
var unpacked int64
|
||||
for i, f := range zr.File {
|
||||
name := strings.TrimPrefix(names[i], prefix)
|
||||
if name == "" || hiddenPath(name) {
|
||||
continue
|
||||
}
|
||||
if f.FileInfo().IsDir() {
|
||||
dirs = append(dirs, strings.TrimSuffix(name, "/"))
|
||||
continue
|
||||
}
|
||||
if !strings.EqualFold(path.Ext(name), ".md") {
|
||||
plan.skip(name, "kein Markdown (Anhänge wie Bilder oder PDFs werden nicht importiert)")
|
||||
continue
|
||||
}
|
||||
if f.UncompressedSize64 > maxImportFileBytes {
|
||||
plan.skip(name, "zu groß (max. 10 MB pro Notiz)")
|
||||
continue
|
||||
}
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
plan.skip(name, "nicht lesbar")
|
||||
continue
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(rc, maxImportFileBytes+1))
|
||||
rc.Close()
|
||||
if err != nil || len(data) > maxImportFileBytes {
|
||||
plan.skip(name, "nicht lesbar oder zu groß")
|
||||
continue
|
||||
}
|
||||
if unpacked += int64(len(data)); unpacked > maxImportUnpackedBytes {
|
||||
return nil, nil, "", errors.New("Archiv ist entpackt zu groß (max. 500 MB)")
|
||||
}
|
||||
if !utf8.Valid(data) {
|
||||
plan.skip(name, "keine UTF-8-Textdatei")
|
||||
continue
|
||||
}
|
||||
notes = append(notes, zipNote{path: name, text: strings.ReplaceAll(string(data), "\r\n", "\n")})
|
||||
}
|
||||
sort.Slice(notes, func(i, j int) bool { return notes[i].path < notes[j].path })
|
||||
return notes, dirs, strings.TrimSuffix(prefix, "/"), nil
|
||||
}
|
||||
|
||||
// cleanArchivePath normalises an entry name and removes anything that could escape (.., absolute paths).
|
||||
func cleanArchivePath(name string) string {
|
||||
name = strings.ReplaceAll(name, "\\", "/")
|
||||
parts := make([]string, 0)
|
||||
for _, p := range strings.Split(name, "/") {
|
||||
if p == "" || p == "." || p == ".." {
|
||||
continue
|
||||
}
|
||||
parts = append(parts, p)
|
||||
}
|
||||
out := strings.Join(parts, "/")
|
||||
if strings.HasSuffix(name, "/") && out != "" {
|
||||
out += "/"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func commonTopFolder(names []string) string {
|
||||
top := ""
|
||||
for _, n := range names {
|
||||
if n == "" || hiddenPath(n) {
|
||||
continue
|
||||
}
|
||||
i := strings.Index(n, "/")
|
||||
if i < 0 {
|
||||
return "" // a file at the root: no common folder
|
||||
}
|
||||
if top == "" {
|
||||
top = n[:i+1]
|
||||
} else if top != n[:i+1] {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
return top
|
||||
}
|
||||
|
||||
// hiddenPath skips Obsidian's settings, trash and OS junk.
|
||||
func hiddenPath(p string) bool {
|
||||
for _, part := range strings.Split(p, "/") {
|
||||
if strings.HasPrefix(part, ".") || part == "__MACOSX" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// --- Planning ---------------------------------------------------------------------------------------
|
||||
|
||||
// PlanObsidianImport reads a vault ZIP. Everything lands in a top folder named after the vault, so an
|
||||
// import never mixes into existing notes. Checkboxes stay checkboxes in their note.
|
||||
func PlanObsidianImport(r io.ReaderAt, size int64, fileName string) (*ImportPlan, error) {
|
||||
plan := &ImportPlan{}
|
||||
notes, dirs, vault, err := readZipNotes(r, size, plan)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if vault == "" {
|
||||
vault = strings.TrimSuffix(path.Base(fileName), path.Ext(fileName))
|
||||
}
|
||||
root := segment(vault)
|
||||
if root == "" {
|
||||
root = "Obsidian-Import"
|
||||
}
|
||||
folders := map[string]bool{root: true}
|
||||
for _, d := range dirs {
|
||||
folders[path.Join(root, folderPathOf(d+"/x"))] = true
|
||||
}
|
||||
for _, n := range notes {
|
||||
fm, body := splitFrontmatter(n.text)
|
||||
folder := path.Join(root, folderPathOf(n.path))
|
||||
folders[folder] = true
|
||||
title := fm.title
|
||||
if title == "" {
|
||||
title = strings.TrimSuffix(path.Base(n.path), path.Ext(n.path))
|
||||
}
|
||||
blocks := MarkdownToBlocks(body)
|
||||
// Front matter tags become hashtags, since MindSift reads tags from the text
|
||||
if extra := missingTags(fm.tags, body); extra != "" {
|
||||
blocks = append(blocks, newBlock("bullet", extra, 0))
|
||||
}
|
||||
plan.Docs = append(plan.Docs, importDoc{folder: folder, title: title, icon: fm.icon, blocks: blocks})
|
||||
}
|
||||
plan.Folders = sortedKeys(folders)
|
||||
if len(plan.Docs) == 0 {
|
||||
return nil, errors.New("Im Archiv wurden keine Markdown-Notizen gefunden")
|
||||
}
|
||||
return plan, nil
|
||||
}
|
||||
|
||||
// PlanMindSiftArchiveImport reads a ZIP created by "Markdown-Archiv herunterladen": notes go back to
|
||||
// their folders, Aufgaben/ back into task lists, Listen/ back into lists.
|
||||
func PlanMindSiftArchiveImport(r io.ReaderAt, size int64) (*ImportPlan, error) {
|
||||
plan := &ImportPlan{}
|
||||
notes, dirs, _, err := readZipNotes(r, size, plan)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
folders := map[string]bool{}
|
||||
for _, d := range dirs {
|
||||
if top := strings.SplitN(d, "/", 2)[0]; top != "Aufgaben" && top != "Listen" {
|
||||
folders[folderPathOf(d+"/x")] = true
|
||||
}
|
||||
}
|
||||
for _, n := range notes {
|
||||
fm, body := splitFrontmatter(n.text)
|
||||
title := fm.title
|
||||
if title == "" {
|
||||
title = strings.TrimSuffix(path.Base(n.path), path.Ext(n.path))
|
||||
}
|
||||
switch folderPathOf(n.path) {
|
||||
case "Aufgaben":
|
||||
tl := importTaskList{title: title, icon: fm.icon, tasks: parseTaskFile(body)}
|
||||
if title == "Ohne Liste" {
|
||||
tl.title = ""
|
||||
}
|
||||
plan.TaskLists = append(plan.TaskLists, tl)
|
||||
case "Listen":
|
||||
plan.Lists = append(plan.Lists, importList{title: title, icon: fm.icon, items: parseListFile(body)})
|
||||
default:
|
||||
folder := folderPathOf(n.path)
|
||||
if folder != "" {
|
||||
folders[folder] = true
|
||||
}
|
||||
plan.Docs = append(plan.Docs, importDoc{folder: folder, title: title, icon: fm.icon, blocks: MarkdownToBlocks(body)})
|
||||
}
|
||||
}
|
||||
plan.Folders = sortedKeys(folders)
|
||||
if len(plan.Docs)+len(plan.TaskLists)+len(plan.Lists) == 0 {
|
||||
return nil, errors.New("Das Archiv enthält keine MindSift-Daten (Notizen, Aufgaben oder Listen)")
|
||||
}
|
||||
return plan, nil
|
||||
}
|
||||
|
||||
// PlanMindSiftJSONImport reads the JSON backup ("Backup herunterladen"). Notes shared with the
|
||||
// exporting user belong to someone else and are skipped; discarded thoughts are not restored.
|
||||
func PlanMindSiftJSONImport(data []byte) (*ImportPlan, error) {
|
||||
var exp domain.FullExport
|
||||
if err := json.Unmarshal(data, &exp); err != nil || (exp.Version == "" && exp.ExportedAt == "") {
|
||||
return nil, errors.New("Die Datei ist kein MindSift-JSON-Backup")
|
||||
}
|
||||
plan := &ImportPlan{}
|
||||
folders := map[string]bool{}
|
||||
for _, d := range exp.Documents {
|
||||
if !d.IsOwner && d.Permission != "" && d.Permission != "owner" {
|
||||
plan.skip(d.Title, "geteilte Notiz eines anderen Benutzers")
|
||||
continue
|
||||
}
|
||||
if d.FolderPath != "" {
|
||||
folders[d.FolderPath] = true
|
||||
}
|
||||
blocks := make([]domain.DocumentBlock, len(d.Blocks))
|
||||
for i, b := range d.Blocks {
|
||||
b.ID, b.DocumentID, b.TaskID = domain.NewID("blk"), "", ""
|
||||
blocks[i] = b
|
||||
}
|
||||
plan.Docs = append(plan.Docs, importDoc{folder: d.FolderPath, title: d.Title, icon: d.Icon, blocks: blocks})
|
||||
}
|
||||
listIndex := map[string]int{}
|
||||
for _, tl := range exp.TaskLists {
|
||||
if !tl.IsOwner {
|
||||
plan.skip(tl.Title, "geteilte Aufgabenliste eines anderen Benutzers")
|
||||
continue
|
||||
}
|
||||
listIndex[tl.ID] = len(plan.TaskLists)
|
||||
plan.TaskLists = append(plan.TaskLists, importTaskList{title: tl.Title, icon: tl.Icon})
|
||||
}
|
||||
for _, t := range exp.Thoughts {
|
||||
task := importTask{content: t.Content, priority: t.Priority, due: t.DueDate, notes: t.Notes, done: t.IsCompleted, subtasks: t.Subtasks}
|
||||
switch {
|
||||
case t.Status == domain.StatusInbox:
|
||||
plan.Inbox = append(plan.Inbox, task)
|
||||
case t.Status == domain.StatusKept || (t.Status == domain.StatusConverted && t.ConvertedType == "task"):
|
||||
if i, ok := listIndex[t.TaskListID]; ok {
|
||||
plan.TaskLists[i].tasks = append(plan.TaskLists[i].tasks, task)
|
||||
} else {
|
||||
plan.TaskLists = append(plan.TaskLists, importTaskList{tasks: []importTask{task}})
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, l := range exp.Lists {
|
||||
if !l.IsOwner {
|
||||
plan.skip(l.Title, "geteilte Liste eines anderen Benutzers")
|
||||
continue
|
||||
}
|
||||
il := importList{title: l.Title, icon: l.Icon}
|
||||
for _, it := range l.Items {
|
||||
il.items = append(il.items, importItem{content: it.Content, category: it.Category, quantity: it.Quantity, done: it.IsCompleted})
|
||||
}
|
||||
plan.Lists = append(plan.Lists, il)
|
||||
}
|
||||
plan.Folders = sortedKeys(folders)
|
||||
return plan, nil
|
||||
}
|
||||
|
||||
// --- Applying ---------------------------------------------------------------------------------------
|
||||
|
||||
// ApplyImport creates everything of the plan for the user. Existing data is never changed; imported
|
||||
// items are added next to it. Task lists of the same name get merged into one new list per import.
|
||||
func (s *Store) ApplyImport(userID string, plan *ImportPlan) (ImportSummary, error) {
|
||||
for _, f := range plan.Folders {
|
||||
_, _ = s.CreateFolder(userID, f) // already existing folders are fine
|
||||
}
|
||||
for _, d := range plan.Docs {
|
||||
title := strings.TrimSpace(d.title)
|
||||
if title == "" {
|
||||
title = "Unbenannte Notiz"
|
||||
}
|
||||
icon := d.icon
|
||||
if icon == "" {
|
||||
icon = "📄"
|
||||
}
|
||||
if _, err := s.CreateDocument(userID, "", title, icon, d.folder, d.blocks, nil, false); err != nil {
|
||||
return plan.Summary(false), fmt.Errorf("Notiz %q: %w", title, err)
|
||||
}
|
||||
}
|
||||
for _, tl := range plan.TaskLists {
|
||||
listID := "" // tasks without a list
|
||||
if tl.title != "" {
|
||||
icon := tl.icon
|
||||
if icon == "" {
|
||||
icon = "📋"
|
||||
}
|
||||
created, err := s.CreateTaskList(userID, "", "", tl.title, icon, "")
|
||||
if err != nil {
|
||||
return plan.Summary(false), fmt.Errorf("Aufgabenliste %q: %w", tl.title, err)
|
||||
}
|
||||
listID = created.ID
|
||||
}
|
||||
for _, t := range tl.tasks {
|
||||
if err := s.importTaskInto(userID, listID, t, true); err != nil {
|
||||
return plan.Summary(false), err
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, t := range plan.Inbox {
|
||||
if err := s.importTaskInto(userID, "", t, false); err != nil {
|
||||
return plan.Summary(false), err
|
||||
}
|
||||
}
|
||||
for _, l := range plan.Lists {
|
||||
icon := l.icon
|
||||
if icon == "" {
|
||||
icon = "🛒"
|
||||
}
|
||||
created, err := s.CreateList(userID, l.title, icon, "shopping")
|
||||
if err != nil {
|
||||
return plan.Summary(false), fmt.Errorf("Liste %q: %w", l.title, err)
|
||||
}
|
||||
for i, it := range l.items {
|
||||
item, err := s.CreateListItem(created.ID, it.content, it.category, it.quantity, i, userID)
|
||||
if err != nil {
|
||||
continue // empty lines and the like
|
||||
}
|
||||
if it.done {
|
||||
_, _ = s.ToggleListItem(item.ID, true, userID, "")
|
||||
}
|
||||
}
|
||||
}
|
||||
return plan.Summary(true), nil
|
||||
}
|
||||
|
||||
func (s *Store) importTaskInto(userID, listID string, t importTask, asTask bool) error {
|
||||
content := strings.TrimSpace(t.content)
|
||||
if content == "" {
|
||||
return nil
|
||||
}
|
||||
priority := t.priority
|
||||
if priority != "low" && priority != "high" && priority != "urgent" {
|
||||
priority = "normal"
|
||||
}
|
||||
th, err := s.CreateThoughtWithDetails(userID, content, priority, "import", t.due, listID, "todo", t.notes)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Aufgabe %q: %w", content, err)
|
||||
}
|
||||
if !asTask {
|
||||
return nil
|
||||
}
|
||||
if _, err := s.UpdateThoughtTriage(userID, th.ID, "keep", "task", "", t.due, listID); err != nil {
|
||||
return fmt.Errorf("Aufgabe %q: %w", content, err)
|
||||
}
|
||||
if t.done || len(t.subtasks) > 0 {
|
||||
var done *bool
|
||||
if t.done {
|
||||
done = &t.done
|
||||
}
|
||||
var subs *[]domain.SubTask
|
||||
if len(t.subtasks) > 0 {
|
||||
subs = &t.subtasks
|
||||
}
|
||||
_, _ = s.UpdateThoughtDetails(userID, th.ID, nil, done, nil, nil, nil, nil, nil, subs, nil)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- Markdown parsing -------------------------------------------------------------------------------
|
||||
|
||||
type frontmatter struct {
|
||||
title, icon string
|
||||
tags []string
|
||||
}
|
||||
|
||||
// splitFrontmatter reads the few keys MindSift uses (title, icon, tags/tag) from YAML front matter.
|
||||
// It is deliberately tolerant: unknown keys and nested structures are ignored.
|
||||
func splitFrontmatter(text string) (frontmatter, string) {
|
||||
var fm frontmatter
|
||||
if !strings.HasPrefix(text, "---\n") {
|
||||
return fm, text
|
||||
}
|
||||
end := strings.Index(text[4:], "\n---")
|
||||
if end < 0 {
|
||||
return fm, text
|
||||
}
|
||||
head := text[4 : 4+end]
|
||||
body := strings.TrimLeft(text[4+end+4:], "\n")
|
||||
key := ""
|
||||
for _, line := range strings.Split(head, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(trimmed, "- ") && (key == "tags" || key == "tag") {
|
||||
fm.tags = append(fm.tags, yamlScalar(trimmed[2:]))
|
||||
continue
|
||||
}
|
||||
k, v, ok := strings.Cut(line, ":")
|
||||
if !ok || strings.HasPrefix(line, " ") {
|
||||
continue
|
||||
}
|
||||
key = strings.TrimSpace(strings.ToLower(k))
|
||||
v = strings.TrimSpace(v)
|
||||
switch key {
|
||||
case "title":
|
||||
fm.title = yamlScalar(v)
|
||||
case "icon":
|
||||
fm.icon = yamlScalar(v)
|
||||
case "tags", "tag":
|
||||
v = strings.Trim(v, "[]")
|
||||
for _, t := range strings.FieldsFunc(v, func(r rune) bool { return r == ',' || r == ' ' }) {
|
||||
if t = yamlScalar(t); t != "" {
|
||||
fm.tags = append(fm.tags, t)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return fm, body
|
||||
}
|
||||
|
||||
func yamlScalar(v string) string {
|
||||
v = strings.TrimSpace(v)
|
||||
if strings.HasPrefix(v, `"`) {
|
||||
var out string
|
||||
if json.Unmarshal([]byte(v), &out) == nil {
|
||||
return out
|
||||
}
|
||||
}
|
||||
return strings.Trim(v, `"'`)
|
||||
}
|
||||
|
||||
func missingTags(tags []string, body string) string {
|
||||
lower := strings.ToLower(body)
|
||||
var out []string
|
||||
for _, t := range tags {
|
||||
t = strings.TrimPrefix(strings.TrimSpace(t), "#")
|
||||
if t == "" || strings.Contains(lower, "#"+strings.ToLower(t)) {
|
||||
continue
|
||||
}
|
||||
out = append(out, "#"+strings.ReplaceAll(t, " ", "_"))
|
||||
}
|
||||
return strings.Join(out, " ")
|
||||
}
|
||||
|
||||
var (
|
||||
listItemRe = regexp.MustCompile(`^([ \t]*)(?:[-*+]|\d+[.)])(?:[ \t]+(.*))?$`)
|
||||
checkboxRe = regexp.MustCompile(`^\[([ xX])\][ \t]*(.*)$`)
|
||||
headingRe = regexp.MustCompile(`^(#{1,6})[ \t]+(.*)$`)
|
||||
fenceRe = regexp.MustCompile("^([ \\t]*)(```|~~~)(.*)$")
|
||||
calloutRe = regexp.MustCompile(`^\[![\w-]+\][+-]?[ \t]*(.*)$`)
|
||||
)
|
||||
|
||||
func newBlock(typ, content string, level int) domain.DocumentBlock {
|
||||
return domain.DocumentBlock{ID: domain.NewID("blk"), Type: typ, Content: content, Level: level}
|
||||
}
|
||||
|
||||
// indentWidth counts a tab as 4 columns.
|
||||
func indentWidth(s string) int {
|
||||
w := 0
|
||||
for _, r := range s {
|
||||
if r == '\t' {
|
||||
w += 4
|
||||
} else {
|
||||
w++
|
||||
}
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// MarkdownToBlocks turns Markdown into outliner blocks. Lists keep their nesting, headings start a
|
||||
// section whose following content is nested below them (the reverse of the export), callouts and
|
||||
// quotes become callout blocks, fenced code becomes a code block, plain paragraphs become bullets.
|
||||
func MarkdownToBlocks(md string) []domain.DocumentBlock {
|
||||
lines := strings.Split(md, "\n")
|
||||
var blocks []domain.DocumentBlock
|
||||
base := 0 // 1 inside a heading section
|
||||
var indents []int // indent widths of the open list levels
|
||||
inList := false // the previous line belonged to a list item (for continuation lines)
|
||||
listLevel := func(width int) int {
|
||||
for len(indents) > 0 && indents[len(indents)-1] > width {
|
||||
indents = indents[:len(indents)-1]
|
||||
}
|
||||
if len(indents) == 0 || indents[len(indents)-1] < width {
|
||||
indents = append(indents, width)
|
||||
}
|
||||
return base + len(indents) - 1
|
||||
}
|
||||
|
||||
for i := 0; i < len(lines); i++ {
|
||||
line := lines[i]
|
||||
trimmed := strings.TrimSpace(line)
|
||||
|
||||
if trimmed == "" {
|
||||
inList = false
|
||||
continue
|
||||
}
|
||||
|
||||
// Fenced code (also inside lists)
|
||||
if m := fenceRe.FindStringSubmatch(line); m != nil {
|
||||
fence, lang, prefix := m[2], strings.TrimSpace(m[3]), m[1]
|
||||
var code []string
|
||||
for i++; i < len(lines); i++ {
|
||||
l := lines[i]
|
||||
if strings.HasPrefix(strings.TrimSpace(l), fence) {
|
||||
break
|
||||
}
|
||||
code = append(code, strings.TrimPrefix(l, prefix))
|
||||
}
|
||||
level := base
|
||||
if prefix != "" && len(indents) > 0 {
|
||||
level = listLevel(indentWidth(prefix))
|
||||
}
|
||||
b := newBlock("code", strings.Join(code, "\n"), level)
|
||||
b.Lang = lang
|
||||
blocks = append(blocks, b)
|
||||
inList = false
|
||||
continue
|
||||
}
|
||||
|
||||
if m := headingRe.FindStringSubmatch(line); m != nil && !strings.HasPrefix(line, " ") && !strings.HasPrefix(line, "\t") {
|
||||
typ := map[int]string{1: "h1", 2: "h2"}[len(m[1])]
|
||||
if typ == "" {
|
||||
typ = "h3"
|
||||
}
|
||||
blocks = append(blocks, newBlock(typ, strings.TrimSpace(m[2]), 0))
|
||||
base, indents, inList = 1, nil, false
|
||||
continue
|
||||
}
|
||||
|
||||
if trimmed == "---" || trimmed == "***" || trimmed == "___" {
|
||||
if !strings.HasPrefix(line, " ") && !strings.HasPrefix(line, "\t") {
|
||||
blocks = append(blocks, newBlock("divider", "", 0))
|
||||
base, indents, inList = 0, nil, false
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Quotes and Obsidian callouts (> [!note] …): one callout block
|
||||
if strings.HasPrefix(trimmed, ">") && !strings.HasPrefix(line, " ") && !strings.HasPrefix(line, "\t") {
|
||||
var parts []string
|
||||
for ; i < len(lines) && strings.HasPrefix(strings.TrimSpace(lines[i]), ">"); i++ {
|
||||
q := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(lines[i]), ">"))
|
||||
if m := calloutRe.FindStringSubmatch(q); m != nil {
|
||||
if q = strings.TrimSpace(m[1]); q == "" {
|
||||
continue
|
||||
}
|
||||
}
|
||||
parts = append(parts, q)
|
||||
}
|
||||
i--
|
||||
blocks = append(blocks, newBlock("callout", strings.Join(parts, "\n"), base))
|
||||
indents, inList = nil, false
|
||||
continue
|
||||
}
|
||||
|
||||
if m := listItemRe.FindStringSubmatch(line); m != nil {
|
||||
level := listLevel(indentWidth(m[1]))
|
||||
content := m[2]
|
||||
typ := "bullet"
|
||||
checked := false
|
||||
if c := checkboxRe.FindStringSubmatch(content); c != nil {
|
||||
typ, checked, content = "todo", c[1] != " ", c[2]
|
||||
} else if h := headingRe.FindStringSubmatch(content); h != nil {
|
||||
typ = map[int]string{1: "h1", 2: "h2"}[len(h[1])]
|
||||
if typ == "" {
|
||||
typ = "h3"
|
||||
}
|
||||
content = strings.TrimSpace(h[2])
|
||||
} else if strings.TrimSpace(content) == "---" {
|
||||
typ, content = "divider", ""
|
||||
} else if strings.HasPrefix(content, "> ") {
|
||||
typ, content = "callout", strings.TrimPrefix(content, "> ")
|
||||
}
|
||||
b := newBlock(typ, content, level)
|
||||
b.Checked = checked
|
||||
blocks = append(blocks, b)
|
||||
inList = true
|
||||
continue
|
||||
}
|
||||
|
||||
// Indented text right below a list item continues it (soft line break)
|
||||
if inList && (strings.HasPrefix(line, " ") || strings.HasPrefix(line, "\t")) && len(blocks) > 0 {
|
||||
last := &blocks[len(blocks)-1]
|
||||
last.Content += "\n" + strings.TrimPrefix(trimmed, "> ")
|
||||
continue
|
||||
}
|
||||
|
||||
// Plain paragraph (consecutive lines form one block)
|
||||
var para []string
|
||||
for ; i < len(lines); i++ {
|
||||
l := strings.TrimSpace(lines[i])
|
||||
if l == "" || headingRe.MatchString(lines[i]) || listItemRe.MatchString(lines[i]) || fenceRe.MatchString(lines[i]) || strings.HasPrefix(l, ">") {
|
||||
break
|
||||
}
|
||||
para = append(para, l)
|
||||
}
|
||||
i--
|
||||
blocks = append(blocks, newBlock("bullet", strings.Join(para, "\n"), base))
|
||||
indents, inList = nil, false
|
||||
}
|
||||
return blocks
|
||||
}
|
||||
|
||||
var (
|
||||
dueRe = regexp.MustCompile(`[ \t]*📅[ \t]*(\d{4}-\d{2}-\d{2})`)
|
||||
priorityRe = regexp.MustCompile(`[ \t]*(🔺|⏫|🔼|🔽|⏬)`)
|
||||
)
|
||||
|
||||
// parseTaskFile reads a task list in Obsidian Tasks format as written by the export.
|
||||
func parseTaskFile(body string) []importTask {
|
||||
var tasks []importTask
|
||||
for _, line := range strings.Split(body, "\n") {
|
||||
if strings.TrimSpace(line) == "" {
|
||||
continue
|
||||
}
|
||||
indented := strings.HasPrefix(line, "\t") || strings.HasPrefix(line, " ")
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if indented && len(tasks) > 0 {
|
||||
last := &tasks[len(tasks)-1]
|
||||
if strings.HasPrefix(trimmed, ">") {
|
||||
note := strings.TrimSpace(strings.TrimPrefix(trimmed, ">"))
|
||||
if last.notes != "" {
|
||||
last.notes += "\n"
|
||||
}
|
||||
last.notes += note
|
||||
} else if m := listItemRe.FindStringSubmatch(trimmed); m != nil {
|
||||
if c := checkboxRe.FindStringSubmatch(m[2]); c != nil {
|
||||
last.subtasks = append(last.subtasks, domain.SubTask{ID: domain.NewID("st"), Title: c[2], IsCompleted: c[1] != " ", CreatedAt: domain.NowISO()})
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
m := listItemRe.FindStringSubmatch(trimmed)
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
t := importTask{content: m[2], priority: "normal"}
|
||||
if c := checkboxRe.FindStringSubmatch(m[2]); c != nil {
|
||||
t.done, t.content = c[1] != " ", c[2]
|
||||
}
|
||||
if d := dueRe.FindStringSubmatch(t.content); d != nil {
|
||||
t.due = d[1]
|
||||
t.content = dueRe.ReplaceAllString(t.content, "")
|
||||
}
|
||||
if p := priorityRe.FindStringSubmatch(t.content); p != nil {
|
||||
t.priority = map[string]string{"🔺": "urgent", "⏫": "high", "🔼": "high", "🔽": "low", "⏬": "low"}[p[1]]
|
||||
t.content = priorityRe.ReplaceAllString(t.content, "")
|
||||
}
|
||||
t.content = strings.TrimSpace(t.content)
|
||||
tasks = append(tasks, t)
|
||||
}
|
||||
return tasks
|
||||
}
|
||||
|
||||
var quantityRe = regexp.MustCompile(`[ \t]*\(([^()]+)\)[ \t]*$`)
|
||||
|
||||
// parseListFile reads list entries "- [ ] Milch (2L) #Kühlregal".
|
||||
func parseListFile(body string) []importItem {
|
||||
var items []importItem
|
||||
for _, line := range strings.Split(body, "\n") {
|
||||
m := listItemRe.FindStringSubmatch(strings.TrimSpace(line))
|
||||
if m == nil || strings.TrimSpace(m[2]) == "" {
|
||||
continue
|
||||
}
|
||||
it := importItem{content: m[2]}
|
||||
if c := checkboxRe.FindStringSubmatch(m[2]); c != nil {
|
||||
it.done, it.content = c[1] != " ", c[2]
|
||||
}
|
||||
// A trailing #Category is turned into the category by CreateListItem itself
|
||||
if q := quantityRe.FindStringSubmatch(stripTrailingTag(it.content)); q != nil {
|
||||
it.quantity = q[1]
|
||||
it.content = strings.Replace(it.content, q[0], "", 1)
|
||||
}
|
||||
it.content = strings.TrimSpace(it.content)
|
||||
items = append(items, it)
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
func stripTrailingTag(s string) string {
|
||||
if i := strings.LastIndex(s, " #"); i >= 0 && !strings.Contains(s[i+2:], " ") {
|
||||
return s[:i]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// folderPathOf returns the folder of a "/"-separated archive path with every part made safe.
|
||||
func folderPathOf(p string) string {
|
||||
dir := path.Dir(p)
|
||||
if dir == "." || dir == "/" {
|
||||
return ""
|
||||
}
|
||||
parts := make([]string, 0)
|
||||
for _, part := range strings.Split(dir, "/") {
|
||||
if seg := segment(part); seg != "" {
|
||||
parts = append(parts, seg)
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, "/")
|
||||
}
|
||||
|
||||
func sortedKeys(m map[string]bool) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
if k != "" {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
Reference in new issue
Block a user