+FTS5 Rebuild

helpers/imports/fts.go

@@ -1,50 +1,156 @@
 package imports
 
 import (
+	"context"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"sync"
+
 	"github.com/Theodor-Springmann-Stiftung/musenalm/dbmodels"
 	"github.com/pocketbase/pocketbase/core"
+	"github.com/pocketbase/pocketbase/tools/types"
 )
 
+var (
+	fts5Mu               sync.Mutex
+	fts5Running          bool
+	fts5Cancel           context.CancelFunc
+	fts5RestartRequested bool
+)
+
+func StartFTS5Rebuild(app core.App, allowRestart bool) (string, error) {
+	fts5Mu.Lock()
+	if fts5Running {
+		if allowRestart {
+			fts5RestartRequested = true
+			if fts5Cancel != nil {
+				fts5Cancel()
+			}
+			done := getSettingInt(app, "fts5_rebuild_done")
+			total := getSettingInt(app, "fts5_rebuild_total")
+			setFTS5RebuildState(app, "running", "Neuaufbau wird neu gestartet.", done, total, "")
+			fts5Mu.Unlock()
+			return "restarting", nil
+		}
+		fts5Mu.Unlock()
+		return "running", nil
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	fts5Running = true
+	fts5Cancel = cancel
+	fts5RestartRequested = false
+	fts5Mu.Unlock()
+
+	go func() {
+		err := rebuildFTSFromScratchWithContext(app, ctx)
+		fts5Mu.Lock()
+		restart := fts5RestartRequested
+		fts5Running = false
+		fts5Cancel = nil
+		fts5RestartRequested = false
+		fts5Mu.Unlock()
+
+		if restart {
+			_, _ = StartFTS5Rebuild(app, false)
+			return
+		}
+
+		if errors.Is(err, context.Canceled) {
+			return
+		}
+	}()
+
+	return "started", nil
+}
+
 func RebuildFTS(app core.App) error {
-	if err := dbmodels.DeleteFTS5Data(app); err != nil {
+	return rebuildFTSWithContext(app, true, context.Background())
+}
+
+func RebuildFTSFromScratch(app core.App) error {
+	return rebuildFTSFromScratchWithContext(app, context.Background())
+}
+
+func rebuildFTSFromScratchWithContext(app core.App, ctx context.Context) error {
+	setFTS5RebuildState(app, "running", "Neuaufbau wird vorbereitet...", 0, 0, "")
+	if err := checkFTS5Canceled(app, ctx, 0, 0); err != nil {
 		return err
 	}
+	if err := dbmodels.DropFTS5Tables(app); err != nil {
+		setFTS5RebuildState(app, "error", "Neuaufbau fehlgeschlagen.", 0, 0, err.Error())
+		return err
+	}
+	if err := checkFTS5Canceled(app, ctx, 0, 0); err != nil {
+		return err
+	}
+	if err := dbmodels.CreateFTS5Tables(app); err != nil {
+		setFTS5RebuildState(app, "error", "Neuaufbau fehlgeschlagen.", 0, 0, err.Error())
+		return err
+	}
+
+	if err := rebuildFTSWithContext(app, false, ctx); err != nil {
+		return err
+	}
+	updateFTS5RebuildTimestamp(app)
+	return nil
+}
+
+func rebuildFTSWithContext(app core.App, clearExisting bool, ctx context.Context) error {
+	fail := func(err error, done, total int) error {
+		setFTS5RebuildState(app, "error", "Neuaufbau fehlgeschlagen.", done, total, err.Error())
+		return err
+	}
+
+	if clearExisting {
+		if err := dbmodels.DeleteFTS5Data(app); err != nil {
+			return fail(err, 0, 0)
+		}
+	}
 
 	places := []*dbmodels.Place{}
 	if err := app.RecordQuery(dbmodels.PLACES_TABLE).All(&places); err != nil {
-		return err
+		return fail(err, 0, 0)
 	}
 	agents := []*dbmodels.Agent{}
 	if err := app.RecordQuery(dbmodels.AGENTS_TABLE).All(&agents); err != nil {
-		return err
+		return fail(err, 0, 0)
 	}
 	series := []*dbmodels.Series{}
 	if err := app.RecordQuery(dbmodels.SERIES_TABLE).All(&series); err != nil {
-		return err
+		return fail(err, 0, 0)
 	}
 	items := []*dbmodels.Item{}
 	if err := app.RecordQuery(dbmodels.ITEMS_TABLE).All(&items); err != nil {
-		return err
+		return fail(err, 0, 0)
 	}
 	entries := []*dbmodels.Entry{}
 	if err := app.RecordQuery(dbmodels.ENTRIES_TABLE).All(&entries); err != nil {
-		return err
+		return fail(err, 0, 0)
 	}
 	contents := []*dbmodels.Content{}
 	if err := app.RecordQuery(dbmodels.CONTENTS_TABLE).All(&contents); err != nil {
-		return err
+		return fail(err, 0, 0)
 	}
 
 	entriesSeries := []*dbmodels.REntriesSeries{}
 	if err := app.RecordQuery(dbmodels.RelationTableName(dbmodels.ENTRIES_TABLE, dbmodels.SERIES_TABLE)).All(&entriesSeries); err != nil {
-		return err
+		return fail(err, 0, 0)
 	}
 	entriesAgents := []*dbmodels.REntriesAgents{}
 	if err := app.RecordQuery(dbmodels.RelationTableName(dbmodels.ENTRIES_TABLE, dbmodels.AGENTS_TABLE)).All(&entriesAgents); err != nil {
-		return err
+		return fail(err, 0, 0)
 	}
 	contentsAgents := []*dbmodels.RContentsAgents{}
 	if err := app.RecordQuery(dbmodels.RelationTableName(dbmodels.CONTENTS_TABLE, dbmodels.AGENTS_TABLE)).All(&contentsAgents); err != nil {
+		return fail(err, 0, 0)
+	}
+
+	total := len(places) + len(agents) + len(series) + len(items) + len(entries) + len(contents)
+	done := 0
+	setFTS5RebuildState(app, "running", "FTS5-Neuaufbau läuft.", done, total, "")
+	if err := checkFTS5Canceled(app, ctx, done, total); err != nil {
 		return err
 	}
 
@@ -92,27 +198,50 @@ func RebuildFTS(app core.App) error {
 	qc := dbmodels.FTS5InsertQuery(app, dbmodels.CONTENTS_TABLE, dbmodels.CONTENTS_FTS5_FIELDS)
 
 	for _, place := range places {
-		if err := dbmodels.BulkInsertFTS5Place(qp, place); err != nil {
+		if err := checkFTS5Canceled(app, ctx, done, total); err != nil {
 			return err
 		}
+		if err := dbmodels.BulkInsertFTS5Place(qp, place); err != nil {
+			return fail(err, done, total)
+		}
+		done++
+		maybeUpdateFTS5Progress(app, ctx, done, total)
 	}
 	for _, agent := range agents {
-		if err := dbmodels.BulkInsertFTS5Agent(qa, agent); err != nil {
+		if err := checkFTS5Canceled(app, ctx, done, total); err != nil {
 			return err
 		}
+		if err := dbmodels.BulkInsertFTS5Agent(qa, agent); err != nil {
+			return fail(err, done, total)
+		}
+		done++
+		maybeUpdateFTS5Progress(app, ctx, done, total)
 	}
 	for _, s := range series {
-		if err := dbmodels.BulkInsertFTS5Series(qs, s); err != nil {
+		if err := checkFTS5Canceled(app, ctx, done, total); err != nil {
 			return err
 		}
+		if err := dbmodels.BulkInsertFTS5Series(qs, s); err != nil {
+			return fail(err, done, total)
+		}
+		done++
+		maybeUpdateFTS5Progress(app, ctx, done, total)
 	}
 	for _, item := range items {
-		if err := dbmodels.BulkInsertFTS5Item(qi, item); err != nil {
+		if err := checkFTS5Canceled(app, ctx, done, total); err != nil {
 			return err
 		}
+		if err := dbmodels.BulkInsertFTS5Item(qi, item); err != nil {
+			return fail(err, done, total)
+		}
+		done++
+		maybeUpdateFTS5Progress(app, ctx, done, total)
 	}
 
 	for _, entry := range entries {
+		if err := checkFTS5Canceled(app, ctx, done, total); err != nil {
+			return err
+		}
 		entryPlaces := []*dbmodels.Place{}
 		for _, placeId := range entry.Places() {
 			if place := placesById[placeId]; place != nil {
@@ -122,17 +251,145 @@ func RebuildFTS(app core.App) error {
 		entryAgents := entriesAgentsMap[entry.Id]
 		entrySeries := entriesSeriesMap[entry.Id]
 		if err := dbmodels.BulkInsertFTS5Entry(qe, entry, entryPlaces, entryAgents, entrySeries); err != nil {
-			return err
+			return fail(err, done, total)
 		}
+		done++
+		maybeUpdateFTS5Progress(app, ctx, done, total)
 	}
 
 	for _, content := range contents {
+		if err := checkFTS5Canceled(app, ctx, done, total); err != nil {
+			return err
+		}
 		entry := entriesById[content.Entry()]
 		contentAgents := contentsAgentsMap[content.Id]
 		if err := dbmodels.BulkInsertFTS5Content(qc, content, entry, contentAgents); err != nil {
+			return fail(err, done, total)
+		}
+		done++
+		maybeUpdateFTS5Progress(app, ctx, done, total)
+	}
+
+	setFTS5RebuildState(app, "complete", "FTS5-Neuaufbau abgeschlossen.", done, total, "")
+	return nil
+}
+
+func maybeUpdateFTS5Progress(app core.App, ctx context.Context, done, total int) {
+	if total <= 0 {
+		return
+	}
+	if done%100 == 0 || done == total {
+		if err := checkFTS5Canceled(app, ctx, done, total); err != nil {
+			return
+		}
+		setFTS5RebuildState(app, "running", "FTS5-Neuaufbau läuft.", done, total, "")
+	}
+}
+
+func checkFTS5Canceled(app core.App, ctx context.Context, done, total int) error {
+	if ctx == nil {
+		return nil
+	}
+	select {
+	case <-ctx.Done():
+		setFTS5RebuildState(app, "aborted", "Neuaufbau abgebrochen.", done, total, "")
+		return ctx.Err()
+	default:
+		return nil
+	}
+}
+
+func updateFTS5RebuildTimestamp(app core.App) {
+	collection, err := app.FindCollectionByNameOrId(dbmodels.SETTINGS_TABLE)
+	if err != nil {
+		app.Logger().Error("Failed to load settings collection for FTS5 timestamp", "error", err)
+		return
+	}
+
+	var record *core.Record
+	existing, err := dbmodels.Settings_Key(app, "fts5_last_rebuild")
+	if err == nil && existing != nil {
+		record = existing.ProxyRecord()
+	} else if err != nil {
+		if !isRecordNotFound(err) {
+			app.Logger().Error("Failed to load FTS5 timestamp setting", "error", err)
+			return
+		}
+	}
+	if record == nil {
+		record = core.NewRecord(collection)
+	}
+
+	record.Set(dbmodels.KEY_FIELD, "fts5_last_rebuild")
+	record.Set(dbmodels.VALUE_FIELD, types.NowDateTime())
+	if err := app.Save(record); err != nil {
+		app.Logger().Error("Failed to save FTS5 timestamp", "error", err)
+	}
+}
+
+func isRecordNotFound(err error) bool {
+	if err == nil {
+		return false
+	}
+	msg := err.Error()
+	return strings.Contains(msg, "no rows in result set") || strings.Contains(msg, "not found")
+}
+
+func setFTS5RebuildState(app core.App, status, message string, done, total int, errMsg string) {
+	_ = upsertSetting(app, "fts5_rebuild_status", status)
+	_ = upsertSetting(app, "fts5_rebuild_message", message)
+	_ = upsertSetting(app, "fts5_rebuild_done", done)
+	_ = upsertSetting(app, "fts5_rebuild_total", total)
+	if errMsg != "" {
+		_ = upsertSetting(app, "fts5_rebuild_error", errMsg)
+	} else {
+		_ = upsertSetting(app, "fts5_rebuild_error", "")
+	}
+}
+
+func upsertSetting(app core.App, key string, value any) error {
+	collection, err := app.FindCollectionByNameOrId(dbmodels.SETTINGS_TABLE)
+	if err != nil {
+		return err
+	}
+
+	var record *core.Record
+	existing, err := dbmodels.Settings_Key(app, key)
+	if err == nil && existing != nil {
+		record = existing.ProxyRecord()
+	} else if err != nil {
+		if !isRecordNotFound(err) {
 			return err
 		}
 	}
-
-	return nil
+	if record == nil {
+		record = core.NewRecord(collection)
+	}
+	record.Set(dbmodels.KEY_FIELD, key)
+	record.Set(dbmodels.VALUE_FIELD, value)
+	return app.Save(record)
+}
+
+func getSettingInt(app core.App, key string) int {
+	setting, err := dbmodels.Settings_Key(app, key)
+	if err != nil || setting == nil {
+		return 0
+	}
+	switch v := setting.Value().(type) {
+	case float64:
+		return int(v)
+	case int:
+		return v
+	case int64:
+		return int(v)
+	case string:
+		if parsed, err := strconv.Atoi(v); err == nil {
+			return parsed
+		}
+	default:
+		if parsed, err := strconv.Atoi(fmt.Sprintf("%v", v)); err == nil {
+			return parsed
+		}
+	}
+	return 0
 }