Paket B: Globale Suche mit SQLite FTS5, BM25-Ranking, Zugriffsprüfung und Cmd+K-Palette
This commit is contained in:
sandrodach committed
1 parent
633bd6f1ef
commit
71fa53ead3
10 files changed
+1030
-55
No files matched your search
@@ -0,0 +1,369 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/sandrodach/mindsift/internal/domain"
|
||||
)
|
||||
|
||||
// SearchHit represents a single ranked match across documents, tasks/thoughts, and lists.
|
||||
type SearchHit struct {
|
||||
Type string `json:"type"` // "document", "thought", "list_item"
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
Snippet string `json:"snippet"`
|
||||
Score float64 `json:"score,omitempty"`
|
||||
DocID string `json:"docId,omitempty"`
|
||||
BlockID string `json:"blockId,omitempty"`
|
||||
ListID string `json:"listId,omitempty"`
|
||||
Icon string `json:"icon,omitempty"`
|
||||
}
|
||||
|
||||
// SearchResults encapsulates all results for backwards-compatible API serialization.
|
||||
type SearchResults struct {
|
||||
Thoughts []domain.Thought `json:"thoughts"`
|
||||
Documents []domain.Document `json:"documents"`
|
||||
ListItems []domain.ListItem `json:"listItems"`
|
||||
Hits []SearchHit `json:"hits"`
|
||||
}
|
||||
|
||||
var wordTokenRegex = regexp.MustCompile(`[\p{L}\p{N}]+`)
|
||||
|
||||
// PrepareFTSQuery extracts valid words from user input and formats them as prefix match tokens.
|
||||
// E.g. "einkau" -> `"einkau"*`, "Hello World!" -> `"Hello"* "World"*`.
|
||||
func PrepareFTSQuery(raw string) string {
|
||||
matches := wordTokenRegex.FindAllString(strings.TrimSpace(raw), -1)
|
||||
if len(matches) == 0 {
|
||||
return ""
|
||||
}
|
||||
tokens := make([]string, 0, len(matches))
|
||||
for _, m := range matches {
|
||||
clean := strings.ReplaceAll(m, `"`, `""`)
|
||||
tokens = append(tokens, fmt.Sprintf(`"%s"*`, clean))
|
||||
}
|
||||
return strings.Join(tokens, " ")
|
||||
}
|
||||
|
||||
// migrateSearchFTS creates the virtual FTS5 tables and synchronization triggers.
|
||||
func (s *Store) migrateSearchFTS() {
|
||||
tables := []string{
|
||||
`CREATE VIRTUAL TABLE IF NOT EXISTS thoughts_fts USING fts5(
|
||||
thought_id UNINDEXED,
|
||||
content,
|
||||
notes,
|
||||
subtasks,
|
||||
tokenize='unicode61'
|
||||
);`,
|
||||
`CREATE VIRTUAL TABLE IF NOT EXISTS documents_fts USING fts5(
|
||||
doc_id UNINDEXED,
|
||||
block_id UNINDEXED,
|
||||
title,
|
||||
content,
|
||||
tokenize='unicode61'
|
||||
);`,
|
||||
`CREATE VIRTUAL TABLE IF NOT EXISTS list_items_fts USING fts5(
|
||||
item_id UNINDEXED,
|
||||
list_id UNINDEXED,
|
||||
content,
|
||||
category,
|
||||
tokenize='unicode61'
|
||||
);`,
|
||||
}
|
||||
|
||||
for _, stmt := range tables {
|
||||
if _, err := s.db.Exec(stmt); err != nil {
|
||||
slog.Error("Failed to create FTS virtual table", "error", err, "statement", stmt)
|
||||
}
|
||||
}
|
||||
|
||||
triggers := []string{
|
||||
// Thoughts triggers
|
||||
`CREATE TRIGGER IF NOT EXISTS trg_thoughts_ai AFTER INSERT ON thoughts BEGIN
|
||||
INSERT INTO thoughts_fts(thought_id, content, notes, subtasks)
|
||||
VALUES (new.id, new.content, new.notes, new.subtasks);
|
||||
END;`,
|
||||
`CREATE TRIGGER IF NOT EXISTS trg_thoughts_au AFTER UPDATE ON thoughts BEGIN
|
||||
DELETE FROM thoughts_fts WHERE thought_id = old.id;
|
||||
INSERT INTO thoughts_fts(thought_id, content, notes, subtasks)
|
||||
VALUES (new.id, new.content, new.notes, new.subtasks);
|
||||
END;`,
|
||||
`CREATE TRIGGER IF NOT EXISTS trg_thoughts_ad AFTER DELETE ON thoughts BEGIN
|
||||
DELETE FROM thoughts_fts WHERE thought_id = old.id;
|
||||
END;`,
|
||||
|
||||
// Documents triggers
|
||||
`CREATE TRIGGER IF NOT EXISTS trg_documents_ai AFTER INSERT ON documents BEGIN
|
||||
INSERT INTO documents_fts(doc_id, block_id, title, content)
|
||||
VALUES (new.id, '', new.title, '');
|
||||
END;`,
|
||||
`CREATE TRIGGER IF NOT EXISTS trg_documents_au AFTER UPDATE OF title, is_archived ON documents BEGIN
|
||||
DELETE FROM documents_fts WHERE doc_id = old.id AND block_id = '';
|
||||
INSERT INTO documents_fts(doc_id, block_id, title, content)
|
||||
SELECT new.id, '', new.title, '' WHERE new.is_archived = 0;
|
||||
END;`,
|
||||
`CREATE TRIGGER IF NOT EXISTS trg_documents_ad AFTER DELETE ON documents BEGIN
|
||||
DELETE FROM documents_fts WHERE doc_id = old.id;
|
||||
END;`,
|
||||
|
||||
// Blocks triggers (excluding tombstones/deleted blocks)
|
||||
`CREATE TRIGGER IF NOT EXISTS trg_blocks_ai AFTER INSERT ON blocks BEGIN
|
||||
INSERT INTO documents_fts(doc_id, block_id, title, content)
|
||||
SELECT new.document_id, new.id, '', new.content
|
||||
WHERE new.deleted = 0 AND (new.deleted_at = '' OR new.deleted_at IS NULL);
|
||||
END;`,
|
||||
`CREATE TRIGGER IF NOT EXISTS trg_blocks_au AFTER UPDATE OF content, deleted, deleted_at, document_id ON blocks BEGIN
|
||||
DELETE FROM documents_fts WHERE doc_id = old.document_id AND block_id = old.id;
|
||||
INSERT INTO documents_fts(doc_id, block_id, title, content)
|
||||
SELECT new.document_id, new.id, '', new.content
|
||||
WHERE new.deleted = 0 AND (new.deleted_at = '' OR new.deleted_at IS NULL);
|
||||
END;`,
|
||||
`CREATE TRIGGER IF NOT EXISTS trg_blocks_ad AFTER DELETE ON blocks BEGIN
|
||||
DELETE FROM documents_fts WHERE doc_id = old.document_id AND block_id = old.id;
|
||||
END;`,
|
||||
|
||||
// List items triggers
|
||||
`CREATE TRIGGER IF NOT EXISTS trg_list_items_ai AFTER INSERT ON list_items BEGIN
|
||||
INSERT INTO list_items_fts(item_id, list_id, content, category)
|
||||
VALUES (new.id, new.list_id, new.content, new.category);
|
||||
END;`,
|
||||
`CREATE TRIGGER IF NOT EXISTS trg_list_items_au AFTER UPDATE ON list_items BEGIN
|
||||
DELETE FROM list_items_fts WHERE item_id = old.id;
|
||||
INSERT INTO list_items_fts(item_id, list_id, content, category)
|
||||
VALUES (new.id, new.list_id, new.content, new.category);
|
||||
END;`,
|
||||
`CREATE TRIGGER IF NOT EXISTS trg_list_items_ad AFTER DELETE ON list_items BEGIN
|
||||
DELETE FROM list_items_fts WHERE item_id = old.id;
|
||||
END;`,
|
||||
}
|
||||
|
||||
for _, trg := range triggers {
|
||||
if _, err := s.db.Exec(trg); err != nil {
|
||||
slog.Error("Failed to create FTS trigger", "error", err, "statement", trg)
|
||||
}
|
||||
}
|
||||
|
||||
s.backfillSearchFTS()
|
||||
}
|
||||
|
||||
// backfillSearchFTS populates empty FTS indices once on startup.
|
||||
func (s *Store) backfillSearchFTS() {
|
||||
var thoughtCount int
|
||||
_ = s.db.QueryRow(`SELECT count(*) FROM thoughts_fts`).Scan(&thoughtCount)
|
||||
if thoughtCount == 0 {
|
||||
_, err := s.db.Exec(`
|
||||
INSERT INTO thoughts_fts(thought_id, content, notes, subtasks)
|
||||
SELECT id, content, notes, subtasks FROM thoughts;
|
||||
`)
|
||||
if err != nil {
|
||||
slog.Error("Failed to backfill thoughts_fts", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
var docCount int
|
||||
_ = s.db.QueryRow(`SELECT count(*) FROM documents_fts`).Scan(&docCount)
|
||||
if docCount == 0 {
|
||||
_, err := s.db.Exec(`
|
||||
INSERT INTO documents_fts(doc_id, block_id, title, content)
|
||||
SELECT id, '', title, '' FROM documents WHERE is_archived = 0;
|
||||
|
||||
INSERT INTO documents_fts(doc_id, block_id, title, content)
|
||||
SELECT document_id, id, '', content FROM blocks
|
||||
WHERE deleted = 0 AND (deleted_at = '' OR deleted_at IS NULL);
|
||||
`)
|
||||
if err != nil {
|
||||
slog.Error("Failed to backfill documents_fts", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
var listCount int
|
||||
_ = s.db.QueryRow(`SELECT count(*) FROM list_items_fts`).Scan(&listCount)
|
||||
if listCount == 0 {
|
||||
_, err := s.db.Exec(`
|
||||
INSERT INTO list_items_fts(item_id, list_id, content, category)
|
||||
SELECT id, list_id, content, category FROM list_items;
|
||||
`)
|
||||
if err != nil {
|
||||
slog.Error("Failed to backfill list_items_fts", "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SearchGlobal executes an FTS5 full-text search across documents, thoughts/tasks and list items,
|
||||
// filtering results strictly to what the user is authorized to view.
|
||||
func (s *Store) SearchGlobal(userID, query string) (*SearchResults, error) {
|
||||
ftsQuery := PrepareFTSQuery(query)
|
||||
res := &SearchResults{
|
||||
Thoughts: make([]domain.Thought, 0),
|
||||
Documents: make([]domain.Document, 0),
|
||||
ListItems: make([]domain.ListItem, 0),
|
||||
Hits: make([]SearchHit, 0),
|
||||
}
|
||||
|
||||
if ftsQuery == "" {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// 1. Search Documents and Blocks
|
||||
// Weights: 10.0 for title, 1.0 for block content
|
||||
docRows, err := s.db.Query(`
|
||||
SELECT f.doc_id, f.block_id,
|
||||
snippet(documents_fts, -1, '<mark>', '</mark>', '…', 14) AS snip,
|
||||
bm25(documents_fts, 10.0, 1.0) AS score,
|
||||
d.title, d.icon, d.user_id, d.is_archived
|
||||
FROM documents_fts f
|
||||
JOIN documents d ON d.id = f.doc_id
|
||||
WHERE documents_fts MATCH ? AND d.is_archived = 0
|
||||
ORDER BY score ASC
|
||||
LIMIT 40
|
||||
`, ftsQuery)
|
||||
if err == nil {
|
||||
seenDocIDs := make(map[string]bool)
|
||||
for docRows.Next() {
|
||||
var docID, blockID, snip, title, icon, ownerID string
|
||||
var score float64
|
||||
var isArchived int
|
||||
if scanErr := docRows.Scan(&docID, &blockID, &snip, &score, &title, &icon, &ownerID, &isArchived); scanErr == nil {
|
||||
// Access check via DocRole
|
||||
if s.DocRole(userID, docID) == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
hitID := docID
|
||||
if blockID != "" {
|
||||
hitID = docID + ":" + blockID
|
||||
}
|
||||
|
||||
res.Hits = append(res.Hits, SearchHit{
|
||||
Type: "document",
|
||||
ID: hitID,
|
||||
Title: title,
|
||||
Snippet: snip,
|
||||
Score: score,
|
||||
DocID: docID,
|
||||
BlockID: blockID,
|
||||
Icon: icon,
|
||||
})
|
||||
|
||||
if !seenDocIDs[docID] {
|
||||
seenDocIDs[docID] = true
|
||||
if doc, err := s.GetDocumentByID(userID, docID); err == nil && doc != nil {
|
||||
res.Documents = append(res.Documents, *doc)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ = docRows.Close()
|
||||
}
|
||||
|
||||
// 2. Search Thoughts & Tasks
|
||||
// Weights: 5.0 for content, 1.0 for notes, 1.0 for subtasks
|
||||
thoughtRows, err := s.db.Query(`
|
||||
SELECT f.thought_id,
|
||||
snippet(thoughts_fts, -1, '<mark>', '</mark>', '…', 14) AS snip,
|
||||
bm25(thoughts_fts, 5.0, 1.0, 1.0) AS score,
|
||||
t.id, t.user_id, t.content, t.tags, t.priority, t.source, t.status,
|
||||
t.is_completed, t.converted_type, t.converted_ref_id, t.due_date,
|
||||
t.task_list_id, t.kanban_status, t.notes, t.alarm_sent, t.created_at,
|
||||
t.triaged_at, t.subtasks, t.sort_order
|
||||
FROM thoughts_fts f
|
||||
JOIN thoughts t ON t.id = f.thought_id
|
||||
WHERE thoughts_fts MATCH ?
|
||||
AND (t.user_id = ? OR t.task_list_id IN (`+s.accessibleTaskListIDsQuery()+`))
|
||||
ORDER BY score ASC
|
||||
LIMIT 40
|
||||
`, ftsQuery, userID, userID, userID, userID)
|
||||
if err == nil {
|
||||
for thoughtRows.Next() {
|
||||
var thoughtID, snip string
|
||||
var score float64
|
||||
var t domain.Thought
|
||||
var tagsStr, subtasksStr string
|
||||
var isCompleted, alarmSent int
|
||||
var triagedAt, dueDate, convertedType, convertedRefId sql.NullString
|
||||
if scanErr := thoughtRows.Scan(
|
||||
&thoughtID, &snip, &score,
|
||||
&t.ID, &t.UserID, &t.Content, &tagsStr, &t.Priority, &t.Source, &t.Status,
|
||||
&isCompleted, &convertedType, &convertedRefId, &dueDate,
|
||||
&t.TaskListID, &t.KanbanStatus, &t.Notes, &alarmSent, &t.CreatedAt,
|
||||
&triagedAt, &subtasksStr, &t.SortOrder,
|
||||
); scanErr == nil {
|
||||
t.IsCompleted = isCompleted != 0
|
||||
t.AlarmSent = alarmSent != 0
|
||||
t.DueDate = dueDate.String
|
||||
t.ConvertedType = convertedType.String
|
||||
t.ConvertedRefID = convertedRefId.String
|
||||
t.TriagedAt = triagedAt.String
|
||||
|
||||
res.Hits = append(res.Hits, SearchHit{
|
||||
Type: "thought",
|
||||
ID: t.ID,
|
||||
Title: t.Content,
|
||||
Snippet: snip,
|
||||
Score: score,
|
||||
})
|
||||
res.Thoughts = append(res.Thoughts, t)
|
||||
}
|
||||
}
|
||||
_ = thoughtRows.Close()
|
||||
}
|
||||
|
||||
// 3. Search List Items
|
||||
// Weights: 2.0 for content, 1.0 for category
|
||||
accessibleLists, _ := s.GetAllListsForUser(userID)
|
||||
accessibleListIDs := make(map[string]bool, len(accessibleLists))
|
||||
for _, l := range accessibleLists {
|
||||
accessibleListIDs[l.ID] = true
|
||||
}
|
||||
|
||||
if len(accessibleListIDs) > 0 {
|
||||
itemRows, err := s.db.Query(`
|
||||
SELECT f.item_id, f.list_id,
|
||||
snippet(list_items_fts, -1, '<mark>', '</mark>', '…', 14) AS snip,
|
||||
bm25(list_items_fts, 2.0, 1.0) AS score,
|
||||
li.id, li.list_id, li.content, li.category, li.quantity,
|
||||
li.is_completed, li.completed_by, li.completed_by_name,
|
||||
li.sort_order, li.created_at, li.completed_at
|
||||
FROM list_items_fts f
|
||||
JOIN list_items li ON li.id = f.item_id
|
||||
WHERE list_items_fts MATCH ?
|
||||
ORDER BY score ASC
|
||||
LIMIT 40
|
||||
`, ftsQuery)
|
||||
if err == nil {
|
||||
for itemRows.Next() {
|
||||
var itemID, listID, snip string
|
||||
var score float64
|
||||
var li domain.ListItem
|
||||
var isCompleted int
|
||||
if scanErr := itemRows.Scan(
|
||||
&itemID, &listID, &snip, &score,
|
||||
&li.ID, &li.ListID, &li.Content, &li.Category, &li.Quantity,
|
||||
&isCompleted, &li.CompletedBy, &li.CompletedByName,
|
||||
&li.SortOrder, &li.CreatedAt, &li.CompletedAt,
|
||||
); scanErr == nil {
|
||||
if !accessibleListIDs[listID] {
|
||||
continue
|
||||
}
|
||||
li.IsCompleted = isCompleted != 0
|
||||
|
||||
res.Hits = append(res.Hits, SearchHit{
|
||||
Type: "list_item",
|
||||
ID: li.ID,
|
||||
Title: li.Content,
|
||||
Snippet: snip,
|
||||
Score: score,
|
||||
ListID: listID,
|
||||
})
|
||||
res.ListItems = append(res.ListItems, li)
|
||||
}
|
||||
}
|
||||
_ = itemRows.Close()
|
||||
}
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
Reference in new issue
Block a user