feat(backup): daily server and user backups, restore by section (backend)

Admins switch automatic backups on, pick the time and the retention
(default 03:00, 3 days). Each run writes a gzip snapshot of the whole
database (VACUUM INTO, consistent while others write) and one MindSift
backup ZIP per active user below $DATA_DIR/backups, then deletes files
past the retention. A restart after the time catches up on that day.
Admins list, download and trigger runs; users list, download and restore
their own backups.

Import and restore can be limited to sections (notes, tasks, lists,
inbox, themes): restore only empties the chosen sections. Export and
import now carry custom themes. File names are checked against a fixed
pattern, so no other file can be fetched through the API.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/storage/account.go

@@ -3,75 +3,137 @@ package storage
 import (
 	"errors"
 	"fmt"
+	"strings"
 )
 
 // ErrLastAdmin protects the server from losing its last administrator.
 var ErrLastAdmin = errors.New("Der letzte Server-Administrator kann nicht gelöscht werden. Bitte ernenne vorher ein anderes Konto zum Administrator.")
 
-// PurgeUserData deletes everything the user owns – thoughts and tasks, notes (with blocks, history and
-// shares), folders, shared folders, task lists and lists – in one transaction. The account, its
-// settings, themes, API keys and group memberships stay. Other people's tasks in the user's task lists
-// become their own personal tasks, as when a single list is deleted.
-func (s *Store) PurgeUserData(userID string) error {
-	tx, err := s.db.Begin()
-	if err != nil {
-		return err
-	}
-	defer tx.Rollback()
+// Sections of a user's data, for restoring only part of a backup.
+const (
+	SectionNotes  = "notes"  // notes, folders, shared folders, pins, tag pages
+	SectionTasks  = "tasks"  // task lists and tasks
+	SectionLists  = "lists"  // shopping and other lists
+	SectionInbox  = "inbox"  // thoughts in the inbox (and the discarded ones)
+	SectionThemes = "themes" // own CSS themes
+)
 
+// AllSections in a fixed order.
+var AllSections = []string{SectionNotes, SectionTasks, SectionLists, SectionInbox, SectionThemes}
+
+// NormalizeSections keeps known sections; an empty selection means all of them.
+func NormalizeSections(sections []string) map[string]bool {
+	out := map[string]bool{}
+	for _, sec := range sections {
+		sec = strings.ToLower(strings.TrimSpace(sec))
+		for _, known := range AllSections {
+			if sec == known {
+				out[sec] = true
+			}
+		}
+	}
+	if len(out) == 0 {
+		for _, sec := range AllSections {
+			out[sec] = true
+		}
+	}
+	return out
+}
+
+// purgeStmts lists what deleting one section of a user's data takes. Each statement gets the user ID
+// as often as it has placeholders.
+func purgeStmts(section string) []struct {
+	q    string
+	args int
+} {
 	ownDocs := `(SELECT id FROM documents WHERE user_id = ?)`
 	ownTaskLists := `(SELECT id FROM task_lists WHERE user_id = ?)`
 	ownFolders := `(SELECT id FROM workspace_shared_folders WHERE owner_id = ?)`
-	stmts := []struct {
+	type st = struct {
 		q    string
-		args int // how many times userID is passed
-	}{
-		// Task lists: keep the tasks of others, drop shares and the lists
-		{`UPDATE thoughts SET task_list_id = '' WHERE task_list_id IN ` + ownTaskLists + ` AND user_id != ?`, 2},
-		{`DELETE FROM task_list_shares WHERE task_list_id IN ` + ownTaskLists, 1},
-		{`DELETE FROM task_lists WHERE user_id = ?`, 1},
-		// Tasks of others linked to blocks of the user's notes lose the link
-		{`UPDATE thoughts SET converted_type = '', converted_ref_id = '' WHERE converted_type = 'document_block'
-		  AND substr(converted_ref_id, 1, instr(converted_ref_id, ':') - 1) IN ` + ownDocs, 1},
-		{`DELETE FROM thoughts WHERE user_id = ?`, 1},
-		// Shared folders owned by the user
-		{`UPDATE documents SET shared_folder_id = '', sub_folder_path = '' WHERE shared_folder_id IN ` + ownFolders, 1},
-		{`DELETE FROM workspace_folder_mounts WHERE folder_id IN ` + ownFolders, 1},
-		{`DELETE FROM workspace_folder_shares WHERE folder_id IN ` + ownFolders, 1},
-		{`DELETE FROM workspace_folders WHERE shared_folder_id IN ` + ownFolders, 1},
-		{`DELETE FROM workspace_shared_folders WHERE owner_id = ?`, 1},
-		// Notes (history tables cascade with the document)
-		{`DELETE FROM blocks WHERE document_id IN ` + ownDocs, 1},
-		{`DELETE FROM block_tags WHERE document_id IN ` + ownDocs, 1},
-		{`DELETE FROM doc_user_folders WHERE doc_id IN ` + ownDocs, 1},
-		{`DELETE FROM doc_team_shares WHERE doc_id IN ` + ownDocs, 1},
-		{`DELETE FROM doc_shares WHERE doc_id IN ` + ownDocs, 1},
-		{`DELETE FROM documents WHERE user_id = ?`, 1},
-		// The user's own placement of notes, folders, pins and tag pages
-		{`DELETE FROM doc_user_folders WHERE user_id = ?`, 1},
-		{`DELETE FROM doc_pins WHERE user_id = ?`, 1},
-		{`DELETE FROM workspace_folders WHERE user_id = ?`, 1},
-		{`DELETE FROM tag_pages WHERE user_id = ?`, 1},
-		// Lists (items cascade)
-		{`DELETE FROM list_shares WHERE list_id IN (SELECT id FROM lists WHERE user_id = ?)`, 1},
-		{`DELETE FROM lists WHERE user_id = ?`, 1},
+		args int
 	}
-	for _, st := range stmts {
-		args := make([]interface{}, st.args)
-		for i := range args {
-			args[i] = userID
+	switch section {
+	case SectionTasks:
+		return []st{
+			// Task lists: keep the tasks of others, drop shares and the lists
+			{`UPDATE thoughts SET task_list_id = '' WHERE task_list_id IN ` + ownTaskLists + ` AND user_id != ?`, 2},
+			{`DELETE FROM task_list_shares WHERE task_list_id IN ` + ownTaskLists, 1},
+			{`DELETE FROM task_lists WHERE user_id = ?`, 1},
+			{`DELETE FROM thoughts WHERE user_id = ? AND (status = 'kept' OR (status = 'converted' AND converted_type IN ('task', 'document_block')))`, 1},
 		}
-		if _, err := tx.Exec(st.q, args...); err != nil {
-			return fmt.Errorf("Daten löschen: %w", err)
+	case SectionInbox:
+		return []st{{`DELETE FROM thoughts WHERE user_id = ? AND status IN ('inbox', 'discarded')`, 1}}
+	case SectionNotes:
+		return []st{
+			// Tasks of others linked to blocks of the user's notes lose the link
+			{`UPDATE thoughts SET converted_type = '', converted_ref_id = '' WHERE converted_type = 'document_block'
+			  AND substr(converted_ref_id, 1, instr(converted_ref_id, ':') - 1) IN ` + ownDocs, 1},
+			// Shared folders owned by the user
+			{`UPDATE documents SET shared_folder_id = '', sub_folder_path = '' WHERE shared_folder_id IN ` + ownFolders, 1},
+			{`DELETE FROM workspace_folder_mounts WHERE folder_id IN ` + ownFolders, 1},
+			{`DELETE FROM workspace_folder_shares WHERE folder_id IN ` + ownFolders, 1},
+			{`DELETE FROM workspace_folders WHERE shared_folder_id IN ` + ownFolders, 1},
+			{`DELETE FROM workspace_shared_folders WHERE owner_id = ?`, 1},
+			// Notes (history tables cascade with the document)
+			{`DELETE FROM blocks WHERE document_id IN ` + ownDocs, 1},
+			{`DELETE FROM block_tags WHERE document_id IN ` + ownDocs, 1},
+			{`DELETE FROM doc_user_folders WHERE doc_id IN ` + ownDocs, 1},
+			{`DELETE FROM doc_team_shares WHERE doc_id IN ` + ownDocs, 1},
+			{`DELETE FROM doc_shares WHERE doc_id IN ` + ownDocs, 1},
+			{`DELETE FROM documents WHERE user_id = ?`, 1},
+			// The user's own placement of notes, folders, pins and tag pages
+			{`DELETE FROM doc_user_folders WHERE user_id = ?`, 1},
+			{`DELETE FROM doc_pins WHERE user_id = ?`, 1},
+			{`DELETE FROM workspace_folders WHERE user_id = ?`, 1},
+			{`DELETE FROM tag_pages WHERE user_id = ?`, 1},
+		}
+	case SectionLists:
+		return []st{
+			{`DELETE FROM list_shares WHERE list_id IN (SELECT id FROM lists WHERE user_id = ?)`, 1},
+			{`DELETE FROM lists WHERE user_id = ?`, 1}, // items cascade
+		}
+	case SectionThemes:
+		return []st{{`DELETE FROM user_themes WHERE user_id = ?`, 1}}
+	}
+	return nil
+}
+
+// PurgeUserSections deletes the chosen sections of the user's own data in one transaction. Content of
+// others shared with the user stays; other people's tasks in the user's task lists become their own
+// personal tasks, as when a single list is deleted.
+func (s *Store) PurgeUserSections(userID string, sections map[string]bool) error {
+	var stmts []sqlStmt
+	for _, sec := range AllSections {
+		if !sections[sec] {
+			continue
+		}
+		for _, st := range purgeStmts(sec) {
+			args := make([]interface{}, st.args)
+			for i := range args {
+				args[i] = userID
+			}
+			stmts = append(stmts, stmt(st.q, args...))
 		}
 	}
-	if err := tx.Commit(); err != nil {
-		return err
+	if sections[SectionTasks] && sections[SectionInbox] {
+		// Everything else of the thought history (turned into notes or list items) goes as well
+		stmts = append(stmts, stmt(`DELETE FROM thoughts WHERE user_id = ?`, userID))
+	}
+	if err := s.execInTx(stmts...); err != nil {
+		return fmt.Errorf("Daten löschen: %w", err)
 	}
 	s.PruneTaskAssignees()
 	return nil
 }
 
+// PurgeUserData deletes everything the user owns – thoughts and tasks, notes (with blocks, history and
+// shares), folders, shared folders, task lists and lists. The account, its settings, themes, API keys
+// and group memberships stay.
+func (s *Store) PurgeUserData(userID string) error {
+	return s.PurgeUserSections(userID, map[string]bool{SectionNotes: true, SectionTasks: true, SectionLists: true, SectionInbox: true})
+}
+
 // IsLastAdmin reports whether userID is an administrator and no other one exists.
 func (s *Store) IsLastAdmin(userID string) bool {
 	var role string

internal/storage/backup.go

@@ -0,0 +1,167 @@
+package storage
+
+import (
+	"compress/gzip"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+	"time"
+)
+
+// Automatic backups below $DATA_DIR/backups/:
+//   system/mindsift-db-YYYY-MM-DD-HHMM.db.gz        the whole database (VACUUM INTO, consistent while
+//                                                     others write), for the admin
+//   users/<user id>/mindsift-backup-YYYY-MM-DD-HHMM.zip   the same ZIP as "Backup herunterladen"
+// Files older than the retention are deleted after each run. Backups next to the database protect
+// against mistakes, not against a broken disk; copy the folder elsewhere as well.
+
+const (
+	DefaultBackupTime          = "03:00"
+	DefaultBackupRetentionDays = 3
+)
+
+var (
+	backupTimeRe   = regexp.MustCompile(`^([01]\d|2[0-3]):[0-5]\d$`)
+	backupNameRe   = regexp.MustCompile(`^mindsift-(db|backup)-\d{4}-\d{2}-\d{2}-\d{4}\.(db\.gz|zip)$`)
+	ErrBackupName  = errors.New("Unbekannte Sicherung")
+	backupStampFmt = "2006-01-02-1504"
+)
+
+func validBackupTime(v string) bool { return backupTimeRe.MatchString(v) }
+
+// BackupFile is one entry of a backup listing.
+type BackupFile struct {
+	Name      string `json:"name"`
+	Size      int64  `json:"size"`
+	CreatedAt string `json:"createdAt"`
+}
+
+// BackupPath returns the file of a backup in dir, refusing anything that is not one of our names
+// (no path tricks through the API).
+func BackupPath(dir, name string) (string, error) {
+	if !backupNameRe.MatchString(name) {
+		return "", ErrBackupName
+	}
+	p := filepath.Join(dir, name)
+	if _, err := os.Stat(p); err != nil {
+		return "", ErrBackupName
+	}
+	return p, nil
+}
+
+// ListBackups lists the backups in dir, newest first.
+func ListBackups(dir string) ([]BackupFile, error) {
+	entries, err := os.ReadDir(dir)
+	if errors.Is(err, os.ErrNotExist) {
+		return []BackupFile{}, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	out := []BackupFile{}
+	for _, e := range entries {
+		if e.IsDir() || !backupNameRe.MatchString(e.Name()) {
+			continue
+		}
+		info, err := e.Info()
+		if err != nil {
+			continue
+		}
+		out = append(out, BackupFile{Name: e.Name(), Size: info.Size(), CreatedAt: info.ModTime().UTC().Format(time.RFC3339)})
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Name > out[j].Name })
+	return out, nil
+}
+
+// PruneBackups deletes backups in dir older than keepDays.
+func PruneBackups(dir string, keepDays int, now time.Time) error {
+	files, err := ListBackups(dir)
+	if err != nil {
+		return err
+	}
+	cutoff := now.Add(-time.Duration(keepDays) * 24 * time.Hour)
+	for _, f := range files {
+		created, err := time.Parse(time.RFC3339, f.CreatedAt)
+		if err == nil && created.Before(cutoff) {
+			if err := os.Remove(filepath.Join(dir, f.Name)); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// WriteSystemSnapshot writes a gzip-compressed copy of the whole database into dir.
+func (s *Store) WriteSystemSnapshot(dir string, now time.Time) (string, error) {
+	if err := os.MkdirAll(dir, 0o700); err != nil {
+		return "", err
+	}
+	name := "mindsift-db-" + now.Format(backupStampFmt) + ".db.gz"
+	tmp := filepath.Join(dir, ".tmp-"+now.Format("20060102150405.000000")+".db")
+	defer os.Remove(tmp)
+	if _, err := s.db.Exec(`VACUUM INTO ?`, tmp); err != nil {
+		return "", fmt.Errorf("Datenbank-Abzug: %w", err)
+	}
+	if err := gzipFile(tmp, filepath.Join(dir, name)); err != nil {
+		return "", err
+	}
+	return name, nil
+}
+
+// WriteUserBackup writes the user's MindSift backup (ZIP with the JSON export) into dir.
+func (s *Store) WriteUserBackup(userID, dir string, now time.Time) (string, error) {
+	if err := os.MkdirAll(dir, 0o700); err != nil {
+		return "", err
+	}
+	exp, err := s.ExportUserData(userID)
+	if err != nil {
+		return "", err
+	}
+	name := "mindsift-backup-" + now.Format(backupStampFmt) + ".zip"
+	return name, writeFileAtomically(filepath.Join(dir, name), func(w io.Writer) error { return WriteBackupZip(exp, w) })
+}
+
+func gzipFile(src, dst string) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+	return writeFileAtomically(dst, func(w io.Writer) error {
+		gz := gzip.NewWriter(w)
+		if _, err := io.Copy(gz, in); err != nil {
+			return err
+		}
+		return gz.Close()
+	})
+}
+
+// writeFileAtomically writes to a temporary file and renames it, so a half-written backup never
+// shows up in the list.
+func writeFileAtomically(dst string, write func(io.Writer) error) error {
+	tmp := dst + ".part"
+	f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
+	if err != nil {
+		return err
+	}
+	if err := write(f); err != nil {
+		f.Close()
+		os.Remove(tmp)
+		return err
+	}
+	if err := f.Close(); err != nil {
+		os.Remove(tmp)
+		return err
+	}
+	return os.Rename(tmp, dst)
+}
+
+// UserBackupDir is the backup folder of one user below root; user IDs are generated by MindSift.
+func UserBackupDir(root, userID string) string {
+	return filepath.Join(root, "users", strings.ReplaceAll(filepath.Base(userID), "..", ""))
+}

internal/storage/import_markdown.go

@@ -28,12 +28,34 @@ const (
 // ImportPlan is everything an import would create. It is built completely (and validated) before
 // anything is written, so the preview shows exactly what the import does.
 type ImportPlan struct {
-	Folders   []string         `json:"-"`
-	Docs      []importDoc      `json:"-"`
-	TaskLists []importTaskList `json:"-"`
-	Lists     []importList     `json:"-"`
-	Inbox     []importTask     `json:"-"`
-	Skipped   []ImportSkip     `json:"skipped"`
+	Folders   []string           `json:"-"`
+	Docs      []importDoc        `json:"-"`
+	TaskLists []importTaskList   `json:"-"`
+	Lists     []importList       `json:"-"`
+	Inbox     []importTask       `json:"-"`
+	Themes    []domain.UserTheme `json:"-"`
+	Skipped   []ImportSkip       `json:"skipped"`
+}
+
+// Only keeps the chosen sections of the plan (see NormalizeSections).
+func (p *ImportPlan) Only(sections map[string]bool) *ImportPlan {
+	out := &ImportPlan{Skipped: p.Skipped}
+	if sections[SectionNotes] {
+		out.Folders, out.Docs = p.Folders, p.Docs
+	}
+	if sections[SectionTasks] {
+		out.TaskLists = p.TaskLists
+	}
+	if sections[SectionLists] {
+		out.Lists = p.Lists
+	}
+	if sections[SectionInbox] {
+		out.Inbox = p.Inbox
+	}
+	if sections[SectionThemes] {
+		out.Themes = p.Themes
+	}
+	return out
 }
 
 type ImportSkip struct {
@@ -50,6 +72,7 @@ type ImportSummary struct {
 	Lists     int          `json:"lists"`
 	ListItems int          `json:"listItems"`
 	Inbox     int          `json:"inbox"`
+	Themes    int          `json:"themes"`
 	Skipped   []ImportSkip `json:"skipped"`
 	Applied   bool         `json:"applied"`
 }
@@ -85,7 +108,7 @@ func (p *ImportPlan) skip(path, reason string) {
 }
 
 func (p *ImportPlan) Summary(applied bool) ImportSummary {
-	sum := ImportSummary{Folders: len(p.Folders), Notes: len(p.Docs), Lists: len(p.Lists), Inbox: len(p.Inbox), Skipped: p.Skipped, Applied: applied}
+	sum := ImportSummary{Folders: len(p.Folders), Notes: len(p.Docs), Lists: len(p.Lists), Inbox: len(p.Inbox), Themes: len(p.Themes), Skipped: p.Skipped, Applied: applied}
 	for _, tl := range p.TaskLists {
 		if tl.title != "" {
 			sum.TaskLists++
@@ -416,6 +439,7 @@ func PlanMindSiftJSONImport(data []byte) (*ImportPlan, error) {
 		}
 		plan.Lists = append(plan.Lists, il)
 	}
+	plan.Themes = exp.Themes
 	plan.Folders = sortedKeys(folders)
 	return plan, nil
 }
@@ -484,6 +508,11 @@ func (s *Store) ApplyImport(userID string, plan *ImportPlan) (ImportSummary, err
 			}
 		}
 	}
+	for _, th := range plan.Themes {
+		if _, err := s.SaveUserTheme(userID, "", th.Name, th.CSS); err != nil {
+			plan.skip(th.Name, "Theme: "+err.Error())
+		}
+	}
 	return plan.Summary(true), nil
 }
 

internal/storage/system.go

@@ -2,6 +2,7 @@ package storage
 
 import (
 	"encoding/json"
+	"strconv"
 	"strings"
 
 	"github.com/sandrodach/mindsift/internal/domain"
@@ -75,6 +76,7 @@ func (s *Store) ExportUserData(userID string) (*domain.FullExport, error) {
 	docs, _ := s.GetDocuments(userID)
 	lists, _ := s.GetAllListsForUser(userID)
 	taskLists, _ := s.GetTaskLists(userID)
+	themes, _ := s.GetUserThemes(userID)
 
 	return &domain.FullExport{
 		ExportedAt: domain.NowISO(),
@@ -84,6 +86,7 @@ func (s *Store) ExportUserData(userID string) (*domain.FullExport, error) {
 		Documents:  docs,
 		Lists:      lists,
 		TaskLists:  taskLists,
+		Themes:     themes,
 	}, nil
 }
 
@@ -111,11 +114,13 @@ func (s *Store) SeedSMTPDefaultsFromEnv(host, port, user, pass, from string, tls
 
 func (s *Store) GetSystemSettings() (*domain.SystemSettings, error) {
 	settings := &domain.SystemSettings{
-		SharingMode:       SharingOrganizations,
-		AllowRegistration: false,
-		InstanceName:      "MindSift",
-		SMTPPort:          "587",
-		SMTPTLS:           true,
+		SharingMode:         SharingOrganizations,
+		AllowRegistration:   false,
+		InstanceName:        "MindSift",
+		SMTPPort:            "587",
+		SMTPTLS:             true,
+		BackupTime:          DefaultBackupTime,
+		BackupRetentionDays: DefaultBackupRetentionDays,
 	}
 
 	rows, err := s.db.Query(`SELECT key, value FROM system_settings`)
@@ -152,6 +157,16 @@ func (s *Store) GetSystemSettings() (*domain.SystemSettings, error) {
 				settings.SMTPSkipVerify = v == "true"
 			case "allow_private_webhooks":
 				settings.AllowPrivateWebhooks = v == "true"
+			case "backup_enabled":
+				settings.BackupEnabled = v == "true"
+			case "backup_time":
+				if validBackupTime(v) {
+					settings.BackupTime = v
+				}
+			case "backup_retention_days":
+				if n, err := strconv.Atoi(v); err == nil && n > 0 {
+					settings.BackupRetentionDays = n
+				}
 			}
 		}
 	}
@@ -182,6 +197,12 @@ func (s *Store) UpdateSystemSettings(settings *domain.SystemSettings) error {
 	if settings.InstanceName == "" {
 		settings.InstanceName = "MindSift"
 	}
+	if !validBackupTime(settings.BackupTime) {
+		settings.BackupTime = DefaultBackupTime
+	}
+	if settings.BackupRetentionDays < 1 || settings.BackupRetentionDays > 365 {
+		settings.BackupRetentionDays = DefaultBackupRetentionDays
+	}
 	_, err := s.db.Exec(`INSERT INTO system_settings (key, value) VALUES 
 		('sharing_mode', ?),
 		('allow_registration', ?),
@@ -193,7 +214,10 @@ func (s *Store) UpdateSystemSettings(settings *domain.SystemSettings) error {
 		('smtp_from', ?),
 		('smtp_tls', ?),
 		('smtp_skip_verify', ?),
-		('allow_private_webhooks', ?)
+		('allow_private_webhooks', ?),
+		('backup_enabled', ?),
+		('backup_time', ?),
+		('backup_retention_days', ?)
 		ON CONFLICT(key) DO UPDATE SET value = excluded.value`,
 		settings.SharingMode,
 		regStr,
@@ -206,6 +230,9 @@ func (s *Store) UpdateSystemSettings(settings *domain.SystemSettings) error {
 		tlsStr,
 		skipVerifyStr,
 		privateHooksStr,
+		strconv.FormatBool(settings.BackupEnabled),
+		settings.BackupTime,
+		strconv.Itoa(settings.BackupRetentionDays),
 	)
 	return err
 }