370 lines
12 KiB
Go
370 lines
12 KiB
Go
package storage
|
|
|
|
import (
|
|
"database/sql"
|
|
"fmt"
|
|
"log/slog"
|
|
"regexp"
|
|
"strings"
|
|
|
|
"github.com/sandrodach/mindsift/internal/domain"
|
|
)
|
|
|
|
// SearchHit represents a single ranked match across documents, tasks/thoughts, and lists.
|
|
type SearchHit struct {
|
|
Type string `json:"type"` // "document", "thought", "list_item"
|
|
ID string `json:"id"`
|
|
Title string `json:"title"`
|
|
Snippet string `json:"snippet"`
|
|
Score float64 `json:"score,omitempty"`
|
|
DocID string `json:"docId,omitempty"`
|
|
BlockID string `json:"blockId,omitempty"`
|
|
ListID string `json:"listId,omitempty"`
|
|
Icon string `json:"icon,omitempty"`
|
|
}
|
|
|
|
// SearchResults encapsulates all results for backwards-compatible API serialization.
|
|
type SearchResults struct {
|
|
Thoughts []domain.Thought `json:"thoughts"`
|
|
Documents []domain.Document `json:"documents"`
|
|
ListItems []domain.ListItem `json:"listItems"`
|
|
Hits []SearchHit `json:"hits"`
|
|
}
|
|
|
|
var wordTokenRegex = regexp.MustCompile(`[\p{L}\p{N}]+`)
|
|
|
|
// PrepareFTSQuery extracts valid words from user input and formats them as prefix match tokens.
|
|
// E.g. "einkau" -> `"einkau"*`, "Hello World!" -> `"Hello"* "World"*`.
|
|
func PrepareFTSQuery(raw string) string {
|
|
matches := wordTokenRegex.FindAllString(strings.TrimSpace(raw), -1)
|
|
if len(matches) == 0 {
|
|
return ""
|
|
}
|
|
tokens := make([]string, 0, len(matches))
|
|
for _, m := range matches {
|
|
clean := strings.ReplaceAll(m, `"`, `""`)
|
|
tokens = append(tokens, fmt.Sprintf(`"%s"*`, clean))
|
|
}
|
|
return strings.Join(tokens, " ")
|
|
}
|
|
|
|
// migrateSearchFTS creates the virtual FTS5 tables and synchronization triggers.
|
|
func (s *Store) migrateSearchFTS() {
|
|
tables := []string{
|
|
`CREATE VIRTUAL TABLE IF NOT EXISTS thoughts_fts USING fts5(
|
|
thought_id UNINDEXED,
|
|
content,
|
|
notes,
|
|
subtasks,
|
|
tokenize='unicode61'
|
|
);`,
|
|
`CREATE VIRTUAL TABLE IF NOT EXISTS documents_fts USING fts5(
|
|
doc_id UNINDEXED,
|
|
block_id UNINDEXED,
|
|
title,
|
|
content,
|
|
tokenize='unicode61'
|
|
);`,
|
|
`CREATE VIRTUAL TABLE IF NOT EXISTS list_items_fts USING fts5(
|
|
item_id UNINDEXED,
|
|
list_id UNINDEXED,
|
|
content,
|
|
category,
|
|
tokenize='unicode61'
|
|
);`,
|
|
}
|
|
|
|
for _, stmt := range tables {
|
|
if _, err := s.db.Exec(stmt); err != nil {
|
|
slog.Error("Failed to create FTS virtual table", "error", err, "statement", stmt)
|
|
}
|
|
}
|
|
|
|
triggers := []string{
|
|
// Thoughts triggers
|
|
`CREATE TRIGGER IF NOT EXISTS trg_thoughts_ai AFTER INSERT ON thoughts BEGIN
|
|
INSERT INTO thoughts_fts(thought_id, content, notes, subtasks)
|
|
VALUES (new.id, new.content, new.notes, new.subtasks);
|
|
END;`,
|
|
`CREATE TRIGGER IF NOT EXISTS trg_thoughts_au AFTER UPDATE ON thoughts BEGIN
|
|
DELETE FROM thoughts_fts WHERE thought_id = old.id;
|
|
INSERT INTO thoughts_fts(thought_id, content, notes, subtasks)
|
|
VALUES (new.id, new.content, new.notes, new.subtasks);
|
|
END;`,
|
|
`CREATE TRIGGER IF NOT EXISTS trg_thoughts_ad AFTER DELETE ON thoughts BEGIN
|
|
DELETE FROM thoughts_fts WHERE thought_id = old.id;
|
|
END;`,
|
|
|
|
// Documents triggers
|
|
`CREATE TRIGGER IF NOT EXISTS trg_documents_ai AFTER INSERT ON documents BEGIN
|
|
INSERT INTO documents_fts(doc_id, block_id, title, content)
|
|
VALUES (new.id, '', new.title, '');
|
|
END;`,
|
|
`CREATE TRIGGER IF NOT EXISTS trg_documents_au AFTER UPDATE OF title, is_archived ON documents BEGIN
|
|
DELETE FROM documents_fts WHERE doc_id = old.id AND block_id = '';
|
|
INSERT INTO documents_fts(doc_id, block_id, title, content)
|
|
SELECT new.id, '', new.title, '' WHERE new.is_archived = 0;
|
|
END;`,
|
|
`CREATE TRIGGER IF NOT EXISTS trg_documents_ad AFTER DELETE ON documents BEGIN
|
|
DELETE FROM documents_fts WHERE doc_id = old.id;
|
|
END;`,
|
|
|
|
// Blocks triggers (excluding tombstones/deleted blocks)
|
|
`CREATE TRIGGER IF NOT EXISTS trg_blocks_ai AFTER INSERT ON blocks BEGIN
|
|
INSERT INTO documents_fts(doc_id, block_id, title, content)
|
|
SELECT new.document_id, new.id, '', new.content
|
|
WHERE new.deleted = 0 AND (new.deleted_at = '' OR new.deleted_at IS NULL);
|
|
END;`,
|
|
`CREATE TRIGGER IF NOT EXISTS trg_blocks_au AFTER UPDATE OF content, deleted, deleted_at, document_id ON blocks BEGIN
|
|
DELETE FROM documents_fts WHERE doc_id = old.document_id AND block_id = old.id;
|
|
INSERT INTO documents_fts(doc_id, block_id, title, content)
|
|
SELECT new.document_id, new.id, '', new.content
|
|
WHERE new.deleted = 0 AND (new.deleted_at = '' OR new.deleted_at IS NULL);
|
|
END;`,
|
|
`CREATE TRIGGER IF NOT EXISTS trg_blocks_ad AFTER DELETE ON blocks BEGIN
|
|
DELETE FROM documents_fts WHERE doc_id = old.document_id AND block_id = old.id;
|
|
END;`,
|
|
|
|
// List items triggers
|
|
`CREATE TRIGGER IF NOT EXISTS trg_list_items_ai AFTER INSERT ON list_items BEGIN
|
|
INSERT INTO list_items_fts(item_id, list_id, content, category)
|
|
VALUES (new.id, new.list_id, new.content, new.category);
|
|
END;`,
|
|
`CREATE TRIGGER IF NOT EXISTS trg_list_items_au AFTER UPDATE ON list_items BEGIN
|
|
DELETE FROM list_items_fts WHERE item_id = old.id;
|
|
INSERT INTO list_items_fts(item_id, list_id, content, category)
|
|
VALUES (new.id, new.list_id, new.content, new.category);
|
|
END;`,
|
|
`CREATE TRIGGER IF NOT EXISTS trg_list_items_ad AFTER DELETE ON list_items BEGIN
|
|
DELETE FROM list_items_fts WHERE item_id = old.id;
|
|
END;`,
|
|
}
|
|
|
|
for _, trg := range triggers {
|
|
if _, err := s.db.Exec(trg); err != nil {
|
|
slog.Error("Failed to create FTS trigger", "error", err, "statement", trg)
|
|
}
|
|
}
|
|
|
|
s.backfillSearchFTS()
|
|
}
|
|
|
|
// backfillSearchFTS populates empty FTS indices once on startup.
|
|
func (s *Store) backfillSearchFTS() {
|
|
var thoughtCount int
|
|
_ = s.db.QueryRow(`SELECT count(*) FROM thoughts_fts`).Scan(&thoughtCount)
|
|
if thoughtCount == 0 {
|
|
_, err := s.db.Exec(`
|
|
INSERT INTO thoughts_fts(thought_id, content, notes, subtasks)
|
|
SELECT id, content, notes, subtasks FROM thoughts;
|
|
`)
|
|
if err != nil {
|
|
slog.Error("Failed to backfill thoughts_fts", "error", err)
|
|
}
|
|
}
|
|
|
|
var docCount int
|
|
_ = s.db.QueryRow(`SELECT count(*) FROM documents_fts`).Scan(&docCount)
|
|
if docCount == 0 {
|
|
_, err := s.db.Exec(`
|
|
INSERT INTO documents_fts(doc_id, block_id, title, content)
|
|
SELECT id, '', title, '' FROM documents WHERE is_archived = 0;
|
|
|
|
INSERT INTO documents_fts(doc_id, block_id, title, content)
|
|
SELECT document_id, id, '', content FROM blocks
|
|
WHERE deleted = 0 AND (deleted_at = '' OR deleted_at IS NULL);
|
|
`)
|
|
if err != nil {
|
|
slog.Error("Failed to backfill documents_fts", "error", err)
|
|
}
|
|
}
|
|
|
|
var listCount int
|
|
_ = s.db.QueryRow(`SELECT count(*) FROM list_items_fts`).Scan(&listCount)
|
|
if listCount == 0 {
|
|
_, err := s.db.Exec(`
|
|
INSERT INTO list_items_fts(item_id, list_id, content, category)
|
|
SELECT id, list_id, content, category FROM list_items;
|
|
`)
|
|
if err != nil {
|
|
slog.Error("Failed to backfill list_items_fts", "error", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// SearchGlobal executes an FTS5 full-text search across documents, thoughts/tasks and list items,
|
|
// filtering results strictly to what the user is authorized to view.
|
|
func (s *Store) SearchGlobal(userID, query string) (*SearchResults, error) {
|
|
ftsQuery := PrepareFTSQuery(query)
|
|
res := &SearchResults{
|
|
Thoughts: make([]domain.Thought, 0),
|
|
Documents: make([]domain.Document, 0),
|
|
ListItems: make([]domain.ListItem, 0),
|
|
Hits: make([]SearchHit, 0),
|
|
}
|
|
|
|
if ftsQuery == "" {
|
|
return res, nil
|
|
}
|
|
|
|
// 1. Search Documents and Blocks
|
|
// Weights: 10.0 for title, 1.0 for block content
|
|
docRows, err := s.db.Query(`
|
|
SELECT f.doc_id, f.block_id,
|
|
snippet(documents_fts, -1, '<mark>', '</mark>', '…', 14) AS snip,
|
|
bm25(documents_fts, 10.0, 1.0) AS score,
|
|
d.title, d.icon, d.user_id, d.is_archived
|
|
FROM documents_fts f
|
|
JOIN documents d ON d.id = f.doc_id
|
|
WHERE documents_fts MATCH ? AND d.is_archived = 0
|
|
ORDER BY score ASC
|
|
LIMIT 40
|
|
`, ftsQuery)
|
|
if err == nil {
|
|
seenDocIDs := make(map[string]bool)
|
|
for docRows.Next() {
|
|
var docID, blockID, snip, title, icon, ownerID string
|
|
var score float64
|
|
var isArchived int
|
|
if scanErr := docRows.Scan(&docID, &blockID, &snip, &score, &title, &icon, &ownerID, &isArchived); scanErr == nil {
|
|
// Access check via DocRole
|
|
if s.DocRole(userID, docID) == "" {
|
|
continue
|
|
}
|
|
|
|
hitID := docID
|
|
if blockID != "" {
|
|
hitID = docID + ":" + blockID
|
|
}
|
|
|
|
res.Hits = append(res.Hits, SearchHit{
|
|
Type: "document",
|
|
ID: hitID,
|
|
Title: title,
|
|
Snippet: snip,
|
|
Score: score,
|
|
DocID: docID,
|
|
BlockID: blockID,
|
|
Icon: icon,
|
|
})
|
|
|
|
if !seenDocIDs[docID] {
|
|
seenDocIDs[docID] = true
|
|
if doc, err := s.GetDocumentByID(userID, docID); err == nil && doc != nil {
|
|
res.Documents = append(res.Documents, *doc)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
_ = docRows.Close()
|
|
}
|
|
|
|
// 2. Search Thoughts & Tasks
|
|
// Weights: 5.0 for content, 1.0 for notes, 1.0 for subtasks
|
|
thoughtRows, err := s.db.Query(`
|
|
SELECT f.thought_id,
|
|
snippet(thoughts_fts, -1, '<mark>', '</mark>', '…', 14) AS snip,
|
|
bm25(thoughts_fts, 5.0, 1.0, 1.0) AS score,
|
|
t.id, t.user_id, t.content, t.tags, t.priority, t.source, t.status,
|
|
t.is_completed, t.converted_type, t.converted_ref_id, t.due_date,
|
|
t.task_list_id, t.kanban_status, t.notes, t.alarm_sent, t.created_at,
|
|
t.triaged_at, t.subtasks, t.sort_order
|
|
FROM thoughts_fts f
|
|
JOIN thoughts t ON t.id = f.thought_id
|
|
WHERE thoughts_fts MATCH ?
|
|
AND (t.user_id = ? OR t.task_list_id IN (`+s.accessibleTaskListIDsQuery()+`))
|
|
ORDER BY score ASC
|
|
LIMIT 40
|
|
`, ftsQuery, userID, userID, userID, userID)
|
|
if err == nil {
|
|
for thoughtRows.Next() {
|
|
var thoughtID, snip string
|
|
var score float64
|
|
var t domain.Thought
|
|
var tagsStr, subtasksStr string
|
|
var isCompleted, alarmSent int
|
|
var triagedAt, dueDate, convertedType, convertedRefId sql.NullString
|
|
if scanErr := thoughtRows.Scan(
|
|
&thoughtID, &snip, &score,
|
|
&t.ID, &t.UserID, &t.Content, &tagsStr, &t.Priority, &t.Source, &t.Status,
|
|
&isCompleted, &convertedType, &convertedRefId, &dueDate,
|
|
&t.TaskListID, &t.KanbanStatus, &t.Notes, &alarmSent, &t.CreatedAt,
|
|
&triagedAt, &subtasksStr, &t.SortOrder,
|
|
); scanErr == nil {
|
|
t.IsCompleted = isCompleted != 0
|
|
t.AlarmSent = alarmSent != 0
|
|
t.DueDate = dueDate.String
|
|
t.ConvertedType = convertedType.String
|
|
t.ConvertedRefID = convertedRefId.String
|
|
t.TriagedAt = triagedAt.String
|
|
|
|
res.Hits = append(res.Hits, SearchHit{
|
|
Type: "thought",
|
|
ID: t.ID,
|
|
Title: t.Content,
|
|
Snippet: snip,
|
|
Score: score,
|
|
})
|
|
res.Thoughts = append(res.Thoughts, t)
|
|
}
|
|
}
|
|
_ = thoughtRows.Close()
|
|
}
|
|
|
|
// 3. Search List Items
|
|
// Weights: 2.0 for content, 1.0 for category
|
|
accessibleLists, _ := s.GetAllListsForUser(userID)
|
|
accessibleListIDs := make(map[string]bool, len(accessibleLists))
|
|
for _, l := range accessibleLists {
|
|
accessibleListIDs[l.ID] = true
|
|
}
|
|
|
|
if len(accessibleListIDs) > 0 {
|
|
itemRows, err := s.db.Query(`
|
|
SELECT f.item_id, f.list_id,
|
|
snippet(list_items_fts, -1, '<mark>', '</mark>', '…', 14) AS snip,
|
|
bm25(list_items_fts, 2.0, 1.0) AS score,
|
|
li.id, li.list_id, li.content, li.category, li.quantity,
|
|
li.is_completed, li.completed_by, li.completed_by_name,
|
|
li.sort_order, li.created_at, li.completed_at
|
|
FROM list_items_fts f
|
|
JOIN list_items li ON li.id = f.item_id
|
|
WHERE list_items_fts MATCH ?
|
|
ORDER BY score ASC
|
|
LIMIT 40
|
|
`, ftsQuery)
|
|
if err == nil {
|
|
for itemRows.Next() {
|
|
var itemID, listID, snip string
|
|
var score float64
|
|
var li domain.ListItem
|
|
var isCompleted int
|
|
if scanErr := itemRows.Scan(
|
|
&itemID, &listID, &snip, &score,
|
|
&li.ID, &li.ListID, &li.Content, &li.Category, &li.Quantity,
|
|
&isCompleted, &li.CompletedBy, &li.CompletedByName,
|
|
&li.SortOrder, &li.CreatedAt, &li.CompletedAt,
|
|
); scanErr == nil {
|
|
if !accessibleListIDs[listID] {
|
|
continue
|
|
}
|
|
li.IsCompleted = isCompleted != 0
|
|
|
|
res.Hits = append(res.Hits, SearchHit{
|
|
Type: "list_item",
|
|
ID: li.ID,
|
|
Title: li.Content,
|
|
Snippet: snip,
|
|
Score: score,
|
|
ListID: listID,
|
|
})
|
|
res.ListItems = append(res.ListItems, li)
|
|
}
|
|
}
|
|
_ = itemRows.Close()
|
|
}
|
|
}
|
|
|
|
return res, nil
|
|
}
|