musenalm/helpers/imports/fts.go

package imports

import (
	"context"
	"errors"
	"fmt"
	"strconv"
	"strings"
	"sync"

	"github.com/Theodor-Springmann-Stiftung/musenalm/dbmodels"
	"github.com/pocketbase/pocketbase/core"
	"github.com/pocketbase/pocketbase/tools/types"
)

var (
	fts5Mu               sync.Mutex
	fts5Running          bool
	fts5Cancel           context.CancelFunc
	fts5RestartRequested bool
)

func StartFTS5Rebuild(app core.App, allowRestart bool) (string, error) {
	fts5Mu.Lock()
	if fts5Running {
		if allowRestart {
			fts5RestartRequested = true
			if fts5Cancel != nil {
				fts5Cancel()
			}
			done := getSettingInt(app, "fts5_rebuild_done")
			total := getSettingInt(app, "fts5_rebuild_total")
			setFTS5RebuildState(app, "running", "Neuaufbau wird neu gestartet.", done, total, "")
			fts5Mu.Unlock()
			return "restarting", nil
		}
		fts5Mu.Unlock()
		return "running", nil
	}
	ctx, cancel := context.WithCancel(context.Background())
	fts5Running = true
	fts5Cancel = cancel
	fts5RestartRequested = false
	fts5Mu.Unlock()

	go func() {
		err := rebuildFTSFromScratchWithContext(app, ctx)
		fts5Mu.Lock()
		restart := fts5RestartRequested
		fts5Running = false
		fts5Cancel = nil
		fts5RestartRequested = false
		fts5Mu.Unlock()

		if restart {
			_, _ = StartFTS5Rebuild(app, false)
			return
		}

		if errors.Is(err, context.Canceled) {
			return
		}
	}()

	return "started", nil
}

func RebuildFTS(app core.App) error {
	return rebuildFTSWithContext(app, true, context.Background())
}

func RebuildFTSFromScratch(app core.App) error {
	return rebuildFTSFromScratchWithContext(app, context.Background())
}

func rebuildFTSFromScratchWithContext(app core.App, ctx context.Context) error {
	setFTS5RebuildState(app, "running", "Neuaufbau wird vorbereitet...", 0, 0, "")
	if err := checkFTS5Canceled(app, ctx, 0, 0); err != nil {
		return err
	}
	if err := dbmodels.DropFTS5Tables(app); err != nil {
		setFTS5RebuildState(app, "error", "Neuaufbau fehlgeschlagen.", 0, 0, err.Error())
		return err
	}
	if err := checkFTS5Canceled(app, ctx, 0, 0); err != nil {
		return err
	}
	if err := dbmodels.CreateFTS5Tables(app); err != nil {
		setFTS5RebuildState(app, "error", "Neuaufbau fehlgeschlagen.", 0, 0, err.Error())
		return err
	}

	if err := rebuildFTSWithContext(app, false, ctx); err != nil {
		return err
	}
	updateFTS5RebuildTimestamp(app)
	return nil
}

func rebuildFTSWithContext(app core.App, clearExisting bool, ctx context.Context) error {
	fail := func(err error, done, total int) error {
		setFTS5RebuildState(app, "error", "Neuaufbau fehlgeschlagen.", done, total, err.Error())
		return err
	}

	if clearExisting {
		if err := dbmodels.DeleteFTS5Data(app); err != nil {
			return fail(err, 0, 0)
		}
	}

	places := []*dbmodels.Place{}
	if err := app.RecordQuery(dbmodels.PLACES_TABLE).All(&places); err != nil {
		return fail(err, 0, 0)
	}
	agents := []*dbmodels.Agent{}
	if err := app.RecordQuery(dbmodels.AGENTS_TABLE).All(&agents); err != nil {
		return fail(err, 0, 0)
	}
	series := []*dbmodels.Series{}
	if err := app.RecordQuery(dbmodels.SERIES_TABLE).All(&series); err != nil {
		return fail(err, 0, 0)
	}
	items := []*dbmodels.Item{}
	if err := app.RecordQuery(dbmodels.ITEMS_TABLE).All(&items); err != nil {
		return fail(err, 0, 0)
	}
	entries := []*dbmodels.Entry{}
	if err := app.RecordQuery(dbmodels.ENTRIES_TABLE).All(&entries); err != nil {
		return fail(err, 0, 0)
	}
	contents := []*dbmodels.Content{}
	if err := app.RecordQuery(dbmodels.CONTENTS_TABLE).All(&contents); err != nil {
		return fail(err, 0, 0)
	}

	entriesSeries := []*dbmodels.REntriesSeries{}
	if err := app.RecordQuery(dbmodels.RelationTableName(dbmodels.ENTRIES_TABLE, dbmodels.SERIES_TABLE)).All(&entriesSeries); err != nil {
		return fail(err, 0, 0)
	}
	entriesAgents := []*dbmodels.REntriesAgents{}
	if err := app.RecordQuery(dbmodels.RelationTableName(dbmodels.ENTRIES_TABLE, dbmodels.AGENTS_TABLE)).All(&entriesAgents); err != nil {
		return fail(err, 0, 0)
	}
	contentsAgents := []*dbmodels.RContentsAgents{}
	if err := app.RecordQuery(dbmodels.RelationTableName(dbmodels.CONTENTS_TABLE, dbmodels.AGENTS_TABLE)).All(&contentsAgents); err != nil {
		return fail(err, 0, 0)
	}

	total := len(places) + len(agents) + len(series) + len(items) + len(entries) + len(contents)
	done := 0
	setFTS5RebuildState(app, "running", "FTS5-Neuaufbau läuft.", done, total, "")
	if err := checkFTS5Canceled(app, ctx, done, total); err != nil {
		return err
	}

	placesById := map[string]*dbmodels.Place{}
	for _, place := range places {
		placesById[place.Id] = place
	}
	agentsById := map[string]*dbmodels.Agent{}
	for _, agent := range agents {
		agentsById[agent.Id] = agent
	}
	seriesById := map[string]*dbmodels.Series{}
	for _, s := range series {
		seriesById[s.Id] = s
	}
	entriesById := map[string]*dbmodels.Entry{}
	for _, entry := range entries {
		entriesById[entry.Id] = entry
	}

	entriesSeriesMap := map[string][]*dbmodels.Series{}
	for _, rel := range entriesSeries {
		if series := seriesById[rel.Series()]; series != nil {
			entriesSeriesMap[rel.Entry()] = append(entriesSeriesMap[rel.Entry()], series)
		}
	}
	entriesAgentsMap := map[string][]*dbmodels.Agent{}
	for _, rel := range entriesAgents {
		if agent := agentsById[rel.Agent()]; agent != nil {
			entriesAgentsMap[rel.Entry()] = append(entriesAgentsMap[rel.Entry()], agent)
		}
	}
	contentsAgentsMap := map[string][]*dbmodels.Agent{}
	for _, rel := range contentsAgents {
		if agent := agentsById[rel.Agent()]; agent != nil {
			contentsAgentsMap[rel.Content()] = append(contentsAgentsMap[rel.Content()], agent)
		}
	}

	qp := dbmodels.FTS5InsertQuery(app, dbmodels.PLACES_TABLE, dbmodels.PLACES_FTS5_FIELDS)
	qa := dbmodels.FTS5InsertQuery(app, dbmodels.AGENTS_TABLE, dbmodels.AGENTS_FTS5_FIELDS)
	qs := dbmodels.FTS5InsertQuery(app, dbmodels.SERIES_TABLE, dbmodels.SERIES_FTS5_FIELDS)
	qi := dbmodels.FTS5InsertQuery(app, dbmodels.ITEMS_TABLE, dbmodels.ITEMS_FTS5_FIELDS)
	qe := dbmodels.FTS5InsertQuery(app, dbmodels.ENTRIES_TABLE, dbmodels.ENTRIES_FTS5_FIELDS)
	qc := dbmodels.FTS5InsertQuery(app, dbmodels.CONTENTS_TABLE, dbmodels.CONTENTS_FTS5_FIELDS)

	for _, place := range places {
		if err := checkFTS5Canceled(app, ctx, done, total); err != nil {
			return err
		}
		if err := dbmodels.BulkInsertFTS5Place(qp, place); err != nil {
			return fail(err, done, total)
		}
		done++
		maybeUpdateFTS5Progress(app, ctx, done, total)
	}
	for _, agent := range agents {
		if err := checkFTS5Canceled(app, ctx, done, total); err != nil {
			return err
		}
		if err := dbmodels.BulkInsertFTS5Agent(qa, agent); err != nil {
			return fail(err, done, total)
		}
		done++
		maybeUpdateFTS5Progress(app, ctx, done, total)
	}
	for _, s := range series {
		if err := checkFTS5Canceled(app, ctx, done, total); err != nil {
			return err
		}
		if err := dbmodels.BulkInsertFTS5Series(qs, s); err != nil {
			return fail(err, done, total)
		}
		done++
		maybeUpdateFTS5Progress(app, ctx, done, total)
	}
	for _, item := range items {
		if err := checkFTS5Canceled(app, ctx, done, total); err != nil {
			return err
		}
		if err := dbmodels.BulkInsertFTS5Item(qi, item); err != nil {
			return fail(err, done, total)
		}
		done++
		maybeUpdateFTS5Progress(app, ctx, done, total)
	}

	for _, entry := range entries {
		if err := checkFTS5Canceled(app, ctx, done, total); err != nil {
			return err
		}
		entryPlaces := []*dbmodels.Place{}
		for _, placeId := range entry.Places() {
			if place := placesById[placeId]; place != nil {
				entryPlaces = append(entryPlaces, place)
			}
		}
		entryAgents := entriesAgentsMap[entry.Id]
		entrySeries := entriesSeriesMap[entry.Id]
		if err := dbmodels.BulkInsertFTS5Entry(qe, entry, entryPlaces, entryAgents, entrySeries); err != nil {
			return fail(err, done, total)
		}
		done++
		maybeUpdateFTS5Progress(app, ctx, done, total)
	}

	for _, content := range contents {
		if err := checkFTS5Canceled(app, ctx, done, total); err != nil {
			return err
		}
		entry := entriesById[content.Entry()]
		contentAgents := contentsAgentsMap[content.Id]
		if err := dbmodels.BulkInsertFTS5Content(qc, content, entry, contentAgents); err != nil {
			return fail(err, done, total)
		}
		done++
		maybeUpdateFTS5Progress(app, ctx, done, total)
	}

	setFTS5RebuildState(app, "complete", "FTS5-Neuaufbau abgeschlossen.", done, total, "")
	return nil
}

func maybeUpdateFTS5Progress(app core.App, ctx context.Context, done, total int) {
	if total <= 0 {
		return
	}
	if done%100 == 0 || done == total {
		if err := checkFTS5Canceled(app, ctx, done, total); err != nil {
			return
		}
		setFTS5RebuildState(app, "running", "FTS5-Neuaufbau läuft.", done, total, "")
	}
}

func checkFTS5Canceled(app core.App, ctx context.Context, done, total int) error {
	if ctx == nil {
		return nil
	}
	select {
	case <-ctx.Done():
		setFTS5RebuildState(app, "aborted", "Neuaufbau abgebrochen.", done, total, "")
		return ctx.Err()
	default:
		return nil
	}
}

func updateFTS5RebuildTimestamp(app core.App) {
	collection, err := app.FindCollectionByNameOrId(dbmodels.SETTINGS_TABLE)
	if err != nil {
		app.Logger().Error("Failed to load settings collection for FTS5 timestamp", "error", err)
		return
	}

	var record *core.Record
	existing, err := dbmodels.Settings_Key(app, "fts5_last_rebuild")
	if err == nil && existing != nil {
		record = existing.ProxyRecord()
	} else if err != nil {
		if !isRecordNotFound(err) {
			app.Logger().Error("Failed to load FTS5 timestamp setting", "error", err)
			return
		}
	}
	if record == nil {
		record = core.NewRecord(collection)
	}

	record.Set(dbmodels.KEY_FIELD, "fts5_last_rebuild")
	record.Set(dbmodels.VALUE_FIELD, types.NowDateTime())
	if err := app.Save(record); err != nil {
		app.Logger().Error("Failed to save FTS5 timestamp", "error", err)
	}
}

func isRecordNotFound(err error) bool {
	if err == nil {
		return false
	}
	msg := err.Error()
	return strings.Contains(msg, "no rows in result set") || strings.Contains(msg, "not found")
}

func setFTS5RebuildState(app core.App, status, message string, done, total int, errMsg string) {
	_ = upsertSetting(app, "fts5_rebuild_status", status)
	_ = upsertSetting(app, "fts5_rebuild_message", message)
	_ = upsertSetting(app, "fts5_rebuild_done", done)
	_ = upsertSetting(app, "fts5_rebuild_total", total)
	if errMsg != "" {
		_ = upsertSetting(app, "fts5_rebuild_error", errMsg)
	} else {
		_ = upsertSetting(app, "fts5_rebuild_error", "")
	}
}

func upsertSetting(app core.App, key string, value any) error {
	collection, err := app.FindCollectionByNameOrId(dbmodels.SETTINGS_TABLE)
	if err != nil {
		return err
	}

	var record *core.Record
	existing, err := dbmodels.Settings_Key(app, key)
	if err == nil && existing != nil {
		record = existing.ProxyRecord()
	} else if err != nil {
		if !isRecordNotFound(err) {
			return err
		}
	}
	if record == nil {
		record = core.NewRecord(collection)
	}
	record.Set(dbmodels.KEY_FIELD, key)
	record.Set(dbmodels.VALUE_FIELD, value)
	return app.Save(record)
}

func getSettingInt(app core.App, key string) int {
	setting, err := dbmodels.Settings_Key(app, key)
	if err != nil || setting == nil {
		return 0
	}
	switch v := setting.Value().(type) {
	case float64:
		return int(v)
	case int:
		return v
	case int64:
		return int(v)
	case string:
		if parsed, err := strconv.Atoi(v); err == nil {
			return parsed
		}
	default:
		if parsed, err := strconv.Atoi(fmt.Sprintf("%v", v)); err == nil {
			return parsed
		}
	}
	return 0
}