feat(workspace): link previews above bullets with a web address (M5)

- GET /link-preview?url= fetches the page on the server (https only,
  SSRF-guarded like webhooks), reads Open Graph/Twitter tags, title,
  description and icon from the head, loads picture and icon (no SVG) and
  keeps them in link_previews (migration 6) for 7 days; readers' browsers
  never contact the page. Off in the demo; 300 fetches per address in
  15 minutes.
- Compact card above the bullet (icon, site, title, one line, small
  picture); the duplicate link badge is left out.
- ✕ turns the address into [title](url), so hiding syncs with the note;
  "Link-Vorschauen anzeigen" under appearance switches all off per device.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/api/docsync.go

@@ -271,6 +271,7 @@ func (s *Server) startCompaction() {
 func (s *Server) compactOnce() {
 	s.leases.Sweep()
 	s.compactMedia(time.Now())
+	_ = s.store.PruneLinkPreviews(time.Now().Add(-30 * 24 * time.Hour))
 	st, err := s.store.CompactDocs(time.Now())
 	if err != nil {
 		slog.Error("Verdichtung des Änderungsprotokolls fehlgeschlagen", "error", err)

internal/api/linkpreview.go

@@ -0,0 +1,294 @@
+package api
+
+import (
+	"context"
+	"errors"
+	"html"
+	"io"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+	"unicode/utf8"
+
+	"github.com/go-chi/chi/v5"
+	"github.com/sandrodach/mindsift/internal/storage"
+)
+
+// Link previews for notes: the server fetches the page once (https only, never into its own network,
+// like alarm webhooks), reads title, description, picture and icon, and keeps them. Readers' browsers
+// only talk to MindSift. Off in the demo, which must not send anything out.
+
+const (
+	previewTTL        = 7 * 24 * time.Hour
+	previewFailTTL    = 24 * time.Hour
+	previewMaxHTML    = 1 << 20
+	previewMaxImage   = 2 << 20
+	previewMaxIcon    = 100 << 10
+	previewPerAddress = 300 // fetches per address in 15 minutes (cached answers do not count)
+)
+
+var previewLocks sync.Map // URL -> *sync.Mutex, one fetch per page at a time
+
+var (
+	metaTagRe  = regexp.MustCompile(`(?is)<meta\s[^>]*>`)
+	linkTagRe  = regexp.MustCompile(`(?is)<link\s[^>]*>`)
+	titleTagRe = regexp.MustCompile(`(?is)<title[^>]*>(.*?)</title>`)
+	attrRe     = regexp.MustCompile(`(?is)([a-z:_-]+)\s*=\s*("([^"]*)"|'([^']*)'|([^\s>]+))`)
+	headEndRe  = regexp.MustCompile(`(?i)</head>|<body[\s>]`)
+	spacesRe   = regexp.MustCompile(`\s+`)
+)
+
+func tagAttrs(tag string) map[string]string {
+	out := map[string]string{}
+	for _, m := range attrRe.FindAllStringSubmatch(tag, -1) {
+		v := m[3]
+		if v == "" {
+			v = m[4]
+		}
+		if v == "" {
+			v = m[5]
+		}
+		out[strings.ToLower(m[1])] = html.UnescapeString(v)
+	}
+	return out
+}
+
+func cleanText(s string, max int) string {
+	s = strings.TrimSpace(spacesRe.ReplaceAllString(html.UnescapeString(s), " "))
+	if utf8.RuneCountInString(s) > max {
+		r := []rune(s)
+		s = strings.TrimSpace(string(r[:max-1])) + "…"
+	}
+	return s
+}
+
+// latin1ToUTF8 for pages that still declare ISO-8859-1/windows-1252.
+func latin1ToUTF8(b []byte) string {
+	r := make([]rune, len(b))
+	for i, c := range b {
+		r[i] = rune(c)
+	}
+	return string(r)
+}
+
+// pageMeta reads the head of an HTML page: Open Graph/Twitter tags, <title>, icon link.
+type pageMeta struct {
+	title, description, siteName, image, icon string
+}
+
+func parsePageMeta(doc string) pageMeta {
+	if loc := headEndRe.FindStringIndex(doc); loc != nil {
+		doc = doc[:loc[0]]
+	}
+	var m pageMeta
+	for _, tag := range metaTagRe.FindAllString(doc, -1) {
+		a := tagAttrs(tag)
+		key := strings.ToLower(a["property"])
+		if key == "" {
+			key = strings.ToLower(a["name"])
+		}
+		v := a["content"]
+		switch key {
+		case "og:title", "twitter:title":
+			if m.title == "" {
+				m.title = v
+			}
+		case "og:description", "twitter:description", "description":
+			if m.description == "" {
+				m.description = v
+			}
+		case "og:site_name", "application-name":
+			if m.siteName == "" {
+				m.siteName = v
+			}
+		case "og:image", "og:image:url", "og:image:secure_url", "twitter:image":
+			if m.image == "" {
+				m.image = v
+			}
+		}
+	}
+	if m.title == "" {
+		if t := titleTagRe.FindStringSubmatch(doc); t != nil {
+			m.title = t[1]
+		}
+	}
+	for _, tag := range linkTagRe.FindAllString(doc, -1) {
+		a := tagAttrs(tag)
+		rel := " " + strings.ToLower(a["rel"]) + " "
+		if (strings.Contains(rel, " icon ") || strings.Contains(rel, "apple-touch-icon")) && a["href"] != "" && !strings.HasSuffix(strings.ToLower(a["href"]), ".svg") {
+			if m.icon == "" || strings.Contains(rel, "apple-touch-icon") {
+				m.icon = a["href"]
+			}
+		}
+	}
+	m.title, m.description, m.siteName = cleanText(m.title, 200), cleanText(m.description, 300), cleanText(m.siteName, 80)
+	return m
+}
+
+// fetchLimited GETs a public https URL and reads at most limit bytes.
+func fetchLimited(ctx context.Context, raw string, limit int64, accept string) ([]byte, http.Header, error) {
+	if err := validateWebhookURL(raw, false, false); err != nil {
+		return nil, nil, err
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, raw, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	req.Header.Set("Accept", accept)
+	req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; MindSift link preview)")
+	req.Header.Set("Accept-Language", "de,en;q=0.8")
+	resp, err := guardedClient(false).Do(req)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, nil, errors.New(resp.Status)
+	}
+	data, err := io.ReadAll(io.LimitReader(resp.Body, limit+1))
+	if err != nil {
+		return nil, nil, err
+	}
+	if int64(len(data)) > limit {
+		return nil, nil, errors.New("zu groß")
+	}
+	return data, resp.Header, nil
+}
+
+func resolveRef(base *url.URL, ref string) string {
+	u, err := base.Parse(strings.TrimSpace(ref))
+	if err != nil || (u.Scheme != "https" && u.Scheme != "http") {
+		return ""
+	}
+	if u.Scheme == "http" {
+		u.Scheme = "https" // only https is fetched; most sites answer there as well
+	}
+	return u.String()
+}
+
+// buildLinkPreview fetches the page and its pictures.
+func buildLinkPreview(ctx context.Context, pageURL string) *storage.LinkPreview {
+	p := &storage.LinkPreview{URL: pageURL}
+	base, err := url.Parse(pageURL)
+	if err != nil {
+		return p
+	}
+	body, hdr, err := fetchLimited(ctx, pageURL, previewMaxHTML, "text/html,application/xhtml+xml")
+	if err != nil {
+		return p
+	}
+	ct := strings.ToLower(hdr.Get("Content-Type"))
+	if !strings.Contains(ct, "html") {
+		// A direct link to an image: the image itself is the preview
+		if _, kind, ok := storage.DetectMedia(body, ""); ok && kind == "image" && len(body) <= previewMaxImage {
+			p.Image, p.ImageMime, p.OK = body, http.DetectContentType(body), true
+			p.Title = cleanText(strings.TrimSuffix(base.Path[strings.LastIndex(base.Path, "/")+1:], "/"), 200)
+			p.SiteName = base.Hostname()
+		}
+		return p
+	}
+	text := string(body)
+	if strings.Contains(ct, "iso-8859-1") || strings.Contains(ct, "windows-1252") || !utf8.Valid(body) {
+		text = latin1ToUTF8(body)
+	}
+	meta := parsePageMeta(text)
+	p.Title, p.Description, p.SiteName = meta.title, meta.description, meta.siteName
+	if p.SiteName == "" {
+		p.SiteName = strings.TrimPrefix(base.Hostname(), "www.")
+	}
+	if img := resolveRef(base, meta.image); img != "" {
+		if data, _, err := fetchLimited(ctx, img, previewMaxImage, "image/*"); err == nil {
+			if mime, kind, ok := storage.DetectMedia(data, ""); ok && kind == "image" {
+				p.Image, p.ImageMime = data, mime
+			}
+		}
+	}
+	iconURL := resolveRef(base, meta.icon)
+	if iconURL == "" {
+		iconURL = resolveRef(base, "/favicon.ico")
+	}
+	if data, _, err := fetchLimited(ctx, iconURL, previewMaxIcon, "image/*"); err == nil {
+		if mime := http.DetectContentType(data); strings.HasPrefix(mime, "image/") && mime != "image/svg+xml" {
+			p.Icon, p.IconMime = data, mime
+		}
+	}
+	p.OK = p.Title != "" || p.Description != "" || len(p.Image) > 0
+	return p
+}
+
+func previewFresh(p *storage.LinkPreview, now time.Time) bool {
+	at, err := time.Parse(time.RFC3339, p.FetchedAt)
+	if err != nil {
+		return false
+	}
+	ttl := previewTTL
+	if !p.OK {
+		ttl = previewFailTTL
+	}
+	return now.Sub(at) < ttl
+}
+
+// handleLinkPreview: GET /link-preview?url=https://…
+func (s *Server) handleLinkPreview(w http.ResponseWriter, r *http.Request) {
+	raw := strings.TrimSpace(r.URL.Query().Get("url"))
+	u, err := url.Parse(raw)
+	if err != nil || u.Scheme != "https" || u.Host == "" || len(raw) > 2000 {
+		writeError(w, http.StatusUnprocessableEntity, "Vorschau nur für https-Links")
+		return
+	}
+	pageURL := u.String()
+	key := storage.LinkPreviewKey(pageURL)
+	if p, err := s.store.GetLinkPreview(key); err == nil && previewFresh(p, time.Now()) {
+		writeJSON(w, http.StatusOK, p)
+		return
+	}
+
+	lock, _ := previewLocks.LoadOrStore(pageURL, &sync.Mutex{})
+	mu := lock.(*sync.Mutex)
+	mu.Lock()
+	defer func() {
+		mu.Unlock()
+		previewLocks.Delete(pageURL)
+	}()
+	// Someone else may have fetched it while this request waited
+	if p, err := s.store.GetLinkPreview(key); err == nil && previewFresh(p, time.Now()) {
+		writeJSON(w, http.StatusOK, p)
+		return
+	}
+	if wait := s.loginGuard.throttle("preview", clientIPFrom(r), previewPerAddress); wait > 0 {
+		writeThrottled(w, wait)
+		return
+	}
+	ctx, cancel := context.WithTimeout(r.Context(), 20*time.Second)
+	defer cancel()
+	p := buildLinkPreview(ctx, pageURL)
+	if err := s.store.SaveLinkPreview(p); err != nil {
+		writeError(w, http.StatusInternalServerError, "Vorschau konnte nicht gespeichert werden")
+		return
+	}
+	writeJSON(w, http.StatusOK, p)
+}
+
+// handleLinkPreviewPicture: GET /link-preview/{key}/image or /icon
+func (s *Server) handleLinkPreviewPicture(w http.ResponseWriter, r *http.Request) {
+	p, err := s.store.GetLinkPreview(chi.URLParam(r, "key"))
+	if err != nil {
+		writeError(w, http.StatusNotFound, "Kein Bild")
+		return
+	}
+	data, mime := p.Image, p.ImageMime
+	if chi.URLParam(r, "kind") == "icon" {
+		data, mime = p.Icon, p.IconMime
+	}
+	if len(data) == 0 {
+		writeError(w, http.StatusNotFound, "Kein Bild")
+		return
+	}
+	w.Header().Set("Content-Type", mime)
+	w.Header().Set("Cache-Control", "private, max-age=86400")
+	w.Header().Set("Content-Security-Policy", "default-src 'none'; sandbox")
+	_, _ = w.Write(data)
+}

internal/api/linkpreview_test.go

@@ -0,0 +1,58 @@
+package api
+
+import (
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/sandrodach/mindsift/internal/storage"
+)
+
+func TestParsePageMeta(t *testing.T) {
+	doc := `<!doctype html><html><head>
+	<meta charset="utf-8"><title>Fallback Titel</title>
+	<meta property="og:title" content="Rezept: Käsespätzle &amp; Salat">
+	<meta name="description" content="Schnell   gemacht,
+	für 4 Personen.">
+	<meta property='og:image' content='/img/spaetzle.jpg'>
+	<meta property="og:site_name" content="Kochseite">
+	<link rel="icon" href="/favicon.png"><link rel="stylesheet" href="x.css">
+	</head><body><meta property="og:title" content="nicht im Kopf"></body></html>`
+	m := parsePageMeta(doc)
+	if m.title != "Rezept: Käsespätzle & Salat" || m.description != "Schnell gemacht, für 4 Personen." ||
+		m.siteName != "Kochseite" || m.image != "/img/spaetzle.jpg" || m.icon != "/favicon.png" {
+		t.Fatalf("unexpected meta: %+v", m)
+	}
+	if m := parsePageMeta(`<head><title> Nur Titel </title></head>`); m.title != "Nur Titel" {
+		t.Fatalf("title fallback: %+v", m)
+	}
+	if long := cleanText(strings.Repeat("a", 500), 200); len([]rune(long)) != 200 || !strings.HasSuffix(long, "…") {
+		t.Fatalf("long text not shortened: %d", len(long))
+	}
+}
+
+func TestLinkPreviewEndpoints(t *testing.T) {
+	srv, store, token := setupTestServer(t)
+	router := srv.Router()
+	if rec := sessionCall(router, "GET", "/api/v1/link-preview?url=http://example.com", token, "", ""); rec.Code != http.StatusUnprocessableEntity {
+		t.Fatalf("http link: want 422, got %d", rec.Code)
+	}
+	// A cached preview is answered without fetching; its pictures are served with the login
+	p := &storage.LinkPreview{URL: "https://example.com/", Title: "Beispiel", OK: true, Icon: []byte("\x89PNG\r\n\x1a\n"), IconMime: "image/png"}
+	if err := store.SaveLinkPreview(p); err != nil {
+		t.Fatal(err)
+	}
+	rec := sessionCall(router, "GET", "/api/v1/link-preview?url=https://example.com/", token, "", "")
+	if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"title":"Beispiel"`) || !strings.Contains(rec.Body.String(), `"hasIcon":true`) {
+		t.Fatalf("cached preview: %d %s", rec.Code, rec.Body.String())
+	}
+	if rec := sessionCall(router, "GET", "/api/v1/link-preview/"+p.Key+"/icon", token, "", ""); rec.Code != http.StatusOK || rec.Header().Get("Content-Type") != "image/png" {
+		t.Fatalf("icon: %d", rec.Code)
+	}
+	if rec := sessionCall(router, "GET", "/api/v1/link-preview/"+p.Key+"/image", token, "", ""); rec.Code != http.StatusNotFound {
+		t.Fatalf("missing image: want 404, got %d", rec.Code)
+	}
+	if rec := sessionCall(router, "GET", "/api/v1/link-preview/"+p.Key+"/icon", "", "", ""); rec.Code != http.StatusUnauthorized {
+		t.Fatalf("without login: want 401, got %d", rec.Code)
+	}
+}

internal/api/server.go

@@ -135,6 +135,8 @@ func (s *Server) Router() http.Handler {
 		r.Post("/api/v1/documents/{id}/media", s.handleUploadMedia)
 		r.Post("/api/v1/documents/{id}/media/fetch", s.notInDemo(s.handleFetchMedia))
 		r.Get("/api/v1/media/{id}", s.handleGetMedia)
+		r.Get("/api/v1/link-preview", s.notInDemo(s.handleLinkPreview))
+		r.Get("/api/v1/link-preview/{key}/{kind}", s.handleLinkPreviewPicture)
 		r.Get("/api/v1/auth/sessions", s.handleListSessions)
 		r.Delete("/api/v1/auth/sessions/{id}", s.notInDemo(s.handleEndSession))
 		r.Post("/api/v1/auth/logout", s.handleLogout)