diff --git a/dbmodels/dbdata.go b/dbmodels/dbdata.go index 0f06592..48c8fec 100644 --- a/dbmodels/dbdata.go +++ b/dbmodels/dbdata.go @@ -4,84 +4,6 @@ import "github.com/pocketbase/pocketbase/tools/types" var EDITORSTATE_VALUES = []string{"Unknown", "ToDo", "Seen", "Partially Edited", "Waiting", "Review", "Edited"} -var SERIES_FTS5_FIELDS = []string{ - SERIES_TITLE_FIELD, - SERIES_PSEUDONYMS_FIELD, - ANNOTATION_FIELD, - COMMENT_FIELD, -} - -var AGENTS_FTS5_FIELDS = []string{ - AGENTS_NAME_FIELD, - AGENTS_BIOGRAPHICAL_DATA_FIELD, - AGENTS_PSEUDONYMS_FIELD, - ANNOTATION_FIELD, - COMMENT_FIELD, -} - -var PLACES_FTS5_FIELDS = []string{ - PLACES_NAME_FIELD, - PLACES_PSEUDONYMS_FIELD, - URI_FIELD, - ANNOTATION_FIELD, - COMMENT_FIELD, -} - -var ITEMS_FTS5_FIELDS = []string{ - ITEMS_LOCATION_FIELD, - ITEMS_OWNER_FIELD, - ITEMS_MEDIA_FIELD, - ITEMS_CONDITION_FIELD, - ITEMS_IDENTIFIER_FIELD, - URI_FIELD, - ANNOTATION_FIELD, - COMMENT_FIELD, -} - -var ENTRIES_FTS5_FIELDS = []string{ - PREFERRED_TITLE_FIELD, - VARIANT_TITLE_FIELD, - PARALLEL_TITLE_FIELD, - TITLE_STMT_FIELD, - SUBTITLE_STMT_FIELD, - INCIPIT_STMT_FIELD, - RESPONSIBILITY_STMT_FIELD, - PUBLICATION_STMT_FIELD, - PLACE_STMT_FIELD, - EDITION_FIELD, - YEAR_FIELD, - EXTENT_FIELD, - DIMENSIONS_FIELD, - REFERENCES_FIELD, - PLACES_TABLE, - AGENTS_TABLE, - SERIES_TABLE, - MUSENALMID_FIELD, - ANNOTATION_FIELD, - COMMENT_FIELD, -} - -var CONTENTS_FTS5_FIELDS = []string{ - PREFERRED_TITLE_FIELD, - VARIANT_TITLE_FIELD, - PARALLEL_TITLE_FIELD, - TITLE_STMT_FIELD, - SUBTITLE_STMT_FIELD, - INCIPIT_STMT_FIELD, - RESPONSIBILITY_STMT_FIELD, - PUBLICATION_STMT_FIELD, - PLACE_STMT_FIELD, - YEAR_FIELD, - EXTENT_FIELD, - DIMENSIONS_FIELD, - NUMBERING_FIELD, - ENTRIES_TABLE, - AGENTS_TABLE, - MUSENALMID_FIELD, - ANNOTATION_FIELD, - COMMENT_FIELD, -} - var ITEM_TYPE_VALUES = []string{ "Original", "Reproduktion", @@ -494,8 +416,6 @@ var PUBLIC_VIEW_RULE = types.Pointer("") var PUBLIC_LIST_RULE = types.Pointer("") const ( - FTS5_PREFIX = "fts5_" - PLACES_TABLE = "places" AGENTS_TABLE = "agents" SERIES_TABLE = "series" @@ -552,7 +472,7 @@ const ( MEDIA_TYPE_FIELD = "media_type" CARRIER_TYPE_FIELD = "carrier_type" - REFERENCES_FIELD = "references" + REFERENCES_FIELD = "refs" URI_FIELD = "uri" MUSENALM_BAENDE_STATUS_FIELD = "musenalm_status" diff --git a/dbmodels/fts5.go b/dbmodels/fts5.go new file mode 100644 index 0000000..527bc92 --- /dev/null +++ b/dbmodels/fts5.go @@ -0,0 +1,398 @@ +package dbmodels + +import ( + "errors" + "strconv" + "strings" + + "github.com/Theodor-Springmann-Stiftung/musenalm/helpers/datatypes" + "github.com/pocketbase/dbx" + "github.com/pocketbase/pocketbase/core" +) + +const ( + FTS5_PREFIX = "fts5_" + DIVIDER_STR = "; " +) + +var SERIES_FTS5_FIELDS = []string{ + SERIES_TITLE_FIELD, + SERIES_PSEUDONYMS_FIELD, + REFERENCES_FIELD, + ANNOTATION_FIELD, + COMMENT_FIELD, +} + +var AGENTS_FTS5_FIELDS = []string{ + AGENTS_NAME_FIELD, + AGENTS_BIOGRAPHICAL_DATA_FIELD, + AGENTS_PSEUDONYMS_FIELD, + ANNOTATION_FIELD, + COMMENT_FIELD, +} + +var PLACES_FTS5_FIELDS = []string{ + PLACES_NAME_FIELD, + PLACES_PSEUDONYMS_FIELD, + URI_FIELD, + ANNOTATION_FIELD, + COMMENT_FIELD, +} + +var ITEMS_FTS5_FIELDS = []string{ + ITEMS_LOCATION_FIELD, + ITEMS_OWNER_FIELD, + ITEMS_MEDIA_FIELD, + ITEMS_CONDITION_FIELD, + ITEMS_IDENTIFIER_FIELD, + URI_FIELD, + ANNOTATION_FIELD, + COMMENT_FIELD, +} + +var ENTRIES_FTS5_FIELDS = []string{ + PREFERRED_TITLE_FIELD, + VARIANT_TITLE_FIELD, + PARALLEL_TITLE_FIELD, + TITLE_STMT_FIELD, + SUBTITLE_STMT_FIELD, + INCIPIT_STMT_FIELD, + RESPONSIBILITY_STMT_FIELD, + PUBLICATION_STMT_FIELD, + PLACE_STMT_FIELD, + EDITION_FIELD, + YEAR_FIELD, + EXTENT_FIELD, + DIMENSIONS_FIELD, + REFERENCES_FIELD, + PLACES_TABLE, + AGENTS_TABLE, + SERIES_TABLE, + MUSENALMID_FIELD, + ANNOTATION_FIELD, + COMMENT_FIELD, +} + +var CONTENTS_FTS5_FIELDS = []string{ + PREFERRED_TITLE_FIELD, + VARIANT_TITLE_FIELD, + PARALLEL_TITLE_FIELD, + TITLE_STMT_FIELD, + SUBTITLE_STMT_FIELD, + INCIPIT_STMT_FIELD, + RESPONSIBILITY_STMT_FIELD, + PUBLICATION_STMT_FIELD, + PLACE_STMT_FIELD, + YEAR_FIELD, + EXTENT_FIELD, + DIMENSIONS_FIELD, + NUMBERING_FIELD, + ENTRIES_TABLE, + AGENTS_TABLE, + MUSENALMID_FIELD, + ANNOTATION_FIELD, + COMMENT_FIELD, +} + +func CreateFTS5TableQuery(tablename string, fields ...string) string { + if len(fields) == 0 { + return "" + } + + str := "CREATE VIRTUAL TABLE IF NOT EXISTS " + FTS5TableName(tablename) + " USING fts5(" + ID_FIELD + " UNINDEXED, " + for i, f := range fields { + str += f + if i < len(fields)-1 { + str += ", " + } + } + str += ", tokenize = 'trigram')" + return str +} + +func FTS5TableName(table string) string { + return FTS5_PREFIX + table +} + +func InsertFTS5Agent(app core.App, agent *Agent) error { + query := FTS5InsertQuery(app, AGENTS_TABLE, AGENTS_FTS5_FIELDS) + return BulkInsertFTS5Agent(query, agent) +} + +func BulkInsertFTS5Agent(query *dbx.Query, agent *Agent) error { + return InsertFTS5Record( + query, + agent.Id, + AGENTS_FTS5_FIELDS, + FTS5ValuesAgent(agent)..., + ) +} + +func InsertFTS5Place(app core.App, place *Place) error { + query := FTS5InsertQuery(app, PLACES_TABLE, PLACES_FTS5_FIELDS) + return BulkInsertFTS5Place(query, place) +} + +func BulkInsertFTS5Place(query *dbx.Query, place *Place) error { + return InsertFTS5Record( + query, + place.Id, + PLACES_FTS5_FIELDS, + FTS5ValuesPlace(place)..., + ) +} + +func InsertFTS5Series(app core.App, series *Series) error { + query := FTS5InsertQuery(app, SERIES_TABLE, SERIES_FTS5_FIELDS) + return BulkInsertFTS5Series(query, series) +} + +func BulkInsertFTS5Series(query *dbx.Query, series *Series) error { + return InsertFTS5Record( + query, + series.Id, + SERIES_FTS5_FIELDS, + FTS5ValuesSeries(series)..., + ) +} + +func InsertFTS5Item(app core.App, item *Item) error { + query := FTS5InsertQuery(app, ITEMS_TABLE, ITEMS_FTS5_FIELDS) + return BulkInsertFTS5Item(query, item) +} + +func BulkInsertFTS5Item(query *dbx.Query, item *Item) error { + return InsertFTS5Record( + query, + item.Id, + ITEMS_FTS5_FIELDS, + FTS5ValuesItem(item)..., + ) +} + +func InsertFTS5Entry(app core.App, entry *Entry, places []*Place, agents []*Agent, series []*Series) error { + query := FTS5InsertQuery(app, ENTRIES_TABLE, ENTRIES_FTS5_FIELDS) + return BulkInsertFTS5Entry(query, entry, places, agents, series) +} + +func BulkInsertFTS5Entry(query *dbx.Query, entry *Entry, places []*Place, agents []*Agent, series []*Series) error { + return InsertFTS5Record( + query, + entry.Id, + ENTRIES_FTS5_FIELDS, + FTS5ValuesEntry(entry, places, agents, series)..., + ) +} + +func InsertFTS5Content(app core.App, content *Content, entry *Entry, agents []*Agent) error { + query := FTS5InsertQuery(app, CONTENTS_TABLE, CONTENTS_FTS5_FIELDS) + return BulkInsertFTS5Content(query, content, entry, agents) +} + +func BulkInsertFTS5Content(query *dbx.Query, content *Content, entry *Entry, agents []*Agent) error { + return InsertFTS5Record( + query, + content.Id, + CONTENTS_FTS5_FIELDS, + FTS5ValuesContent(content, entry, agents)..., + ) +} + +func FTS5ValuesContent(content *Content, entry *Entry, agents []*Agent) []string { + agentstring := "" + if agents != nil { + agentstring = datatypes.SliceJoin(agents, DIVIDER_STR, func(agent *Agent) string { + if agent == nil { + return "" + } + return agent.Name() + }) + } + + entrystring := entry.PreferredTitle() + if entry.Year() != 0 { + entrystring += "; " + strconv.Itoa(entry.Year()) + } else { + entrystring += "; [o.J.]" + } + + strconv.Itoa(entry.Year()) + return []string{ + content.PreferredTitle(), + content.VariantTitle(), + content.ParallelTitle(), + content.TitleStmt(), + content.SubtitleStmt(), + content.IncipitStmt(), + content.ResponsibilityStmt(), + content.PublicationStmt(), + content.PlaceStmt(), + strconv.Itoa(content.Year()), + content.Extent(), + content.Dimensions(), + strconv.FormatFloat(content.Numbering(), 'f', 3, 64), + entrystring, + agentstring, + content.MusenalmID(), + datatypes.DeleteTags(content.Annotation()), + datatypes.DeleteTags(content.Comment()), + } +} + +func FTS5ValuesEntry(entry *Entry, places []*Place, agents []*Agent, series []*Series) []string { + placestring := "" + if places != nil { + placestring = datatypes.SliceJoin(places, DIVIDER_STR, func(place *Place) string { + if place == nil { + return "" + } + return place.Name() + }) + } + + agentstring := "" + if agents != nil { + agentstring = datatypes.SliceJoin(agents, DIVIDER_STR, func(agent *Agent) string { + if agent == nil { + return "" + } + return agent.Name() + }) + } + + seriesstring := "" + if series != nil { + seriesstring = datatypes.SliceJoin(series, DIVIDER_STR, func(series *Series) string { + if series == nil { + return "" + } + return series.Title() + }) + } + return []string{ + entry.PreferredTitle(), + entry.VariantTitle(), + entry.ParallelTitle(), + entry.TitleStmt(), + entry.SubtitleStmt(), + entry.IncipitStmt(), + entry.ResponsibilityStmt(), + entry.PublicationStmt(), + entry.PlaceStmt(), + entry.Edition(), + strconv.Itoa(entry.Year()), + entry.Extent(), + entry.Dimensions(), + entry.References(), + placestring, + agentstring, + seriesstring, + entry.MusenalmID(), + datatypes.DeleteTags(entry.Annotation()), + datatypes.DeleteTags(entry.Comment()), + } +} + +func FTS5ValuesItem(item *Item) []string { + return []string{ + item.Location(), + item.Owner(), + strings.Join(item.Media(), DIVIDER_STR), + item.Condition(), + item.Identifier(), + item.Uri(), + datatypes.DeleteTags(item.Annotation()), + datatypes.DeleteTags(item.Comment()), + } +} + +func FTS5ValuesSeries(series *Series) []string { + return []string{ + series.Title(), + series.Pseudonyms(), + series.References(), + datatypes.DeleteTags(series.Annotation()), + datatypes.DeleteTags(series.Comment()), + } +} + +func FTS5ValuesPlace(place *Place) []string { + return []string{ + place.Name(), + place.Pseudonyms(), + place.URI(), + datatypes.DeleteTags(place.Annotation()), + datatypes.DeleteTags(place.Comment()), + } +} + +func FTS5ValuesAgent(agent *Agent) []string { + return []string{ + agent.Name(), + agent.BiographicalData(), + agent.Pseudonyms(), + datatypes.DeleteTags(agent.Annotation()), + datatypes.DeleteTags(agent.Comment()), + } +} + +func FTS5ValuesItems(item *Item) []string { + return []string{ + item.Location(), + item.Owner(), + strings.Join(item.Media(), DIVIDER_STR), + item.Condition(), + item.Identifier(), + item.Uri(), + datatypes.DeleteTags(item.Annotation()), + datatypes.DeleteTags(item.Comment()), + } +} + +func FTS5InsertQuery(app core.App, table string, fields []string) *dbx.Query { + tn := FTS5TableName(table) + query := "INSERT INTO " + + tn + + " (" + + ID_FIELD + + ", " + + strings.Join(fields, ", ") + + ") VALUES ({:" + + ID_FIELD + + "}, {:" + + strings.Join(fields, "}, {:") + + "})" + return app.DB().NewQuery(query).Prepare() +} + +func InsertFTS5Record(query *dbx.Query, id string, fields []string, values ...string) error { + if len(fields) != len(values) { + return errors.New("fields and values must have the same length") + } + + params := dbx.Params{ID_FIELD: id} + for i, v := range fields { + params[v] = values[i] + } + + _, err := query.Bind(params).Execute() + return err +} + +func DeleteFTS5Data(app core.App) error { + err1 := deleteTableContents(app, FTS5TableName(AGENTS_TABLE)) + err2 := deleteTableContents(app, FTS5TableName(SERIES_TABLE)) + err3 := deleteTableContents(app, FTS5TableName(ENTRIES_TABLE)) + err4 := deleteTableContents(app, FTS5TableName(PLACES_TABLE)) + err5 := deleteTableContents(app, FTS5TableName(ITEMS_TABLE)) + err6 := deleteTableContents(app, FTS5TableName(CONTENTS_TABLE)) + return errors.Join(err1, err2, err3, err4, err5, err6) +} + +func deleteTableContents(app core.App, table string) error { + _, err := app.DB().NewQuery("DELETE FROM " + table).Execute() + if err != nil { + return err + } + return nil +} diff --git a/dbmodels/functions.go b/dbmodels/functions.go index ae054de..66e225e 100644 --- a/dbmodels/functions.go +++ b/dbmodels/functions.go @@ -97,19 +97,3 @@ func GetFields(records []*core.Record, field string) []any { } return fields } - -func CreateFTS5TableQuery(tablename string, fields ...string) string { - if len(fields) == 0 { - return "" - } - - str := "CREATE VIRTUAL TABLE IF NOT EXISTS " + FTS5_PREFIX + tablename + " USING fts5(id UNINDEXED, " - for i, f := range fields { - str += f - if i < len(fields)-1 { - str += ", " - } - } - str += " tokenize = 'trigram')" - return str -} diff --git a/dbmodels/item.go b/dbmodels/item.go index d44c094..47118b4 100644 --- a/dbmodels/item.go +++ b/dbmodels/item.go @@ -114,10 +114,10 @@ func (a *Item) SetEditState(editState string) { a.Set(EDITSTATE_FIELD, editState) } -func (a *Item) Comments() string { +func (a *Item) Comment() string { return a.GetString(COMMENT_FIELD) } -func (a *Item) SetComments(comments string) { +func (a *Item) SetComment(comments string) { a.Set(COMMENT_FIELD, comments) } diff --git a/dbmodels/place.go b/dbmodels/place.go index fa69e85..79b42d8 100644 --- a/dbmodels/place.go +++ b/dbmodels/place.go @@ -26,6 +26,14 @@ func (p *Place) SetName(name string) { p.Set(PLACES_NAME_FIELD, name) } +func (p *Place) Pseudonyms() string { + return p.GetString(PLACES_PSEUDONYMS_FIELD) +} + +func (p *Place) SetPseudonyms(pseudonyms string) { + p.Set(PLACES_PSEUDONYMS_FIELD, pseudonyms) +} + func (p *Place) Fictional() bool { return p.GetBool(PLACES_FICTIONAL_FIELD) } diff --git a/helpers/datatypes/slice.go b/helpers/datatypes/slice.go new file mode 100644 index 0000000..54b5a96 --- /dev/null +++ b/helpers/datatypes/slice.go @@ -0,0 +1 @@ +package datatypes diff --git a/helpers/datatypes/string.go b/helpers/datatypes/string.go new file mode 100644 index 0000000..c0b10fb --- /dev/null +++ b/helpers/datatypes/string.go @@ -0,0 +1,30 @@ +package datatypes + +import ( + "regexp" + "strings" +) + +var html_regexp = regexp.MustCompile(`<[^>]+>`) + +func DeleteTags(s string) string { + return html_regexp.ReplaceAllString(s, "") +} + +func NormalizeString(s string) string { + s = strings.TrimSpace(s) + s = strings.ReplaceAll(s, "
", "") + s = strings.ReplaceAll(s, "
", "") + return s +} + +func SliceJoin[T any](slice []T, join string, f func(T) string) string { + var result []string + for _, item := range slice { + ap := f(item) + if ap != "" { + result = append(result, ap) + } + } + return strings.Join(result, join) +} diff --git a/migrations/1739007272_insert_data.go b/migrations/1739007272_insert_data.go index bc143a9..6cfb9ad 100644 --- a/migrations/1739007272_insert_data.go +++ b/migrations/1739007272_insert_data.go @@ -1,6 +1,7 @@ package migrations import ( + "errors" "sync" "github.com/Theodor-Springmann-Stiftung/musenalm/dbmodels" @@ -22,9 +23,18 @@ func init() { var agentsmap map[string]*dbmodels.Agent var placesmap map[string]*dbmodels.Place + var placesmapid map[string]*dbmodels.Place var seriesmap map[string]*dbmodels.Series var entriesmap map[string]*dbmodels.Entry + var entriesmapid map[string]*dbmodels.Entry + var seriesmapid map[string]*dbmodels.Series + var agentsmapid map[string]*dbmodels.Agent var contentsmap map[string]*dbmodels.Content + var r_entries_series map[string][]*dbmodels.REntriesSeries + var r_entries_agents map[string][]*dbmodels.REntriesAgents + var r_contents_agents map[string][]*dbmodels.RContentsAgents + + var items []*dbmodels.Item wg := sync.WaitGroup{} wg.Add(3) @@ -41,53 +51,54 @@ func init() { panic(err) } agentsmap = datatypes.MakeMap(agents, func(record *dbmodels.Agent) string { return record.MusenalmID() }) + agentsmapid = datatypes.MakeMap(agents, func(record *dbmodels.Agent) string { return record.Id }) wg.Done() }() go func() { places, err := seed.RecordsFromOrte(app, adb.Orte) - if err == nil { - for _, record := range places { - if err = app.Save(record); err != nil { - app.Logger().Error("Error saving record", "error", err, "record", record) - } - } - } else { + if err != nil { panic(err) } + for _, record := range places { + if err = app.Save(record); err != nil { + app.Logger().Error("Error saving record", "error", err, "record", record) + } + } placesmap = datatypes.MakeMap(places, func(record *dbmodels.Place) string { return record.Name() }) + placesmapid = datatypes.MakeMap(places, func(record *dbmodels.Place) string { return record.Id }) wg.Done() }() go func() { series, err := seed.RecordsFromReihentitel(app, adb.Reihen) - if err == nil { - for _, record := range series { - if err = app.Save(record); err != nil { - app.Logger().Error("Error saving record", "error", err, "record", record) - } - } - } else { + if err != nil { panic(err) } + for _, record := range series { + if err = app.Save(record); err != nil { + app.Logger().Error("Error saving record", "error", err, "record", record) + } + } seriesmap = datatypes.MakeMap(series, func(record *dbmodels.Series) string { return record.MusenalmID() }) + seriesmapid = datatypes.MakeMap(series, func(record *dbmodels.Series) string { return record.Id }) wg.Done() }() wg.Wait() entries, err := seed.RecordsFromBände(app, *adb, placesmap) - if err == nil { - for _, record := range entries { - if err = app.Save(record); err != nil { - app.Logger().Error("Error saving record", "error", err, "record", record) - } - } - } else { + if err != nil { panic(err) } + for _, record := range entries { + if err = app.Save(record); err != nil { + app.Logger().Error("Error saving record", "error", err, "record", record) + } + } entriesmap = datatypes.MakeMap(entries, func(record *dbmodels.Entry) string { return record.MusenalmID() }) + entriesmapid = datatypes.MakeMap(entries, func(record *dbmodels.Entry) string { return record.Id }) wg.Add(2) @@ -106,15 +117,14 @@ func init() { go func() { contents, err := seed.RecordsFromInhalte(app, adb.Inhalte, entriesmap) - if err == nil { - for _, record := range contents { - if err = app.Save(record); err != nil { - app.Logger().Error("Error saving record", "error", err, "record", record) - } - } - } else { + if err != nil { panic(err) } + for _, record := range contents { + if err = app.Save(record); err != nil { + app.Logger().Error("Error saving record", "error", err, "record", record) + } + } contentsmap = datatypes.MakeMap(contents, func(record *dbmodels.Content) string { return record.MusenalmID() }) wg.Done() }() @@ -124,62 +134,152 @@ func init() { wg.Add(3) go func() { - if records, err := seed.RecordsFromRelationBändeReihen(app, adb.Relationen_Bände_Reihen, seriesmap, entriesmap); err == nil { - for _, record := range records { - if err := app.Save(record); err != nil { - app.Logger().Error("Error saving record", "error", err, "record", record) - } - } - } else { + records, err := seed.RecordsFromRelationBändeReihen(app, adb.Relationen_Bände_Reihen, seriesmap, entriesmap) + if err != nil { panic(err) } + for _, record := range records { + if err = app.Save(record); err != nil { + app.Logger().Error("Error saving record", "error", err, "record", record) + } + } + r_entries_series = datatypes.MakeMultiMap( + records, + func(record *dbmodels.REntriesSeries) string { return record.Entry() }) wg.Done() }() go func() { - if records, err := seed.RecordsFromRelationBändeAkteure(app, adb.Relationen_Bände_Akteure, entriesmap, agentsmap); err == nil { - for _, record := range records { - if err := app.Save(record); err != nil { - app.Logger().Error("Error saving record", "error", err, "record", record) - } - } - } else { + records, err := seed.RecordsFromRelationBändeAkteure(app, adb.Relationen_Bände_Akteure, entriesmap, agentsmap) + if err != nil { panic(err) } + for _, record := range records { + if err = app.Save(record); err != nil { + app.Logger().Error("Error saving record", "error", err, "record", record) + } + } + r_entries_agents = datatypes.MakeMultiMap( + records, + func(record *dbmodels.REntriesAgents) string { return record.Entry() }) wg.Done() }() go func() { - if records, err := seed.RecordsFromRelationInhalteAkteure(app, adb.Relationen_Inhalte_Akteure, contentsmap, agentsmap); err == nil { - for _, record := range records { - if err := app.Save(record); err != nil { - app.Logger().Error("Error saving record", "error", err, "record", record) - } - } - } else { + records, err := seed.RecordsFromRelationInhalteAkteure(app, adb.Relationen_Inhalte_Akteure, contentsmap, agentsmap) + if err != nil { panic(err) } + for _, record := range records { + if err = app.Save(record); err != nil { + app.Logger().Error("Error saving record", "error", err, "record", record) + } + } + r_contents_agents = datatypes.MakeMultiMap( + records, + func(record *dbmodels.RContentsAgents) string { return record.Content() }) wg.Done() }() wg.Wait() + // INFO: Inserting FTS5 data + qp := dbmodels.FTS5InsertQuery(app, dbmodels.PLACES_TABLE, dbmodels.PLACES_FTS5_FIELDS) + qa := dbmodels.FTS5InsertQuery(app, dbmodels.AGENTS_TABLE, dbmodels.AGENTS_FTS5_FIELDS) + qs := dbmodels.FTS5InsertQuery(app, dbmodels.SERIES_TABLE, dbmodels.SERIES_FTS5_FIELDS) + qi := dbmodels.FTS5InsertQuery(app, dbmodels.ITEMS_TABLE, dbmodels.ITEMS_FTS5_FIELDS) + qe := dbmodels.FTS5InsertQuery(app, dbmodels.ENTRIES_TABLE, dbmodels.ENTRIES_FTS5_FIELDS) + qc := dbmodels.FTS5InsertQuery(app, dbmodels.CONTENTS_TABLE, dbmodels.CONTENTS_FTS5_FIELDS) + + for _, place := range placesmap { + if err = dbmodels.BulkInsertFTS5Place(qp, place); err != nil { + app.Logger().Error("Error inserting place", "error", err, "place", place) + } + } + + for _, agent := range agentsmap { + if err = dbmodels.BulkInsertFTS5Agent(qa, agent); err != nil { + app.Logger().Error("Error inserting agent", "error", err, "agent", agent) + } + } + + for _, series := range seriesmap { + if err = dbmodels.BulkInsertFTS5Series(qs, series); err != nil { + app.Logger().Error("Error inserting series", "error", err, "series", series) + } + } + + for _, item := range items { + if err = dbmodels.BulkInsertFTS5Item(qi, item); err != nil { + app.Logger().Error("Error inserting item", "error", err, "item", item) + } + } + + for _, entry := range entriesmap { + places := []*dbmodels.Place{} + for _, place := range entry.Places() { + places = append(places, placesmapid[place]) + } + + ser := []*dbmodels.Series{} + for _, series := range r_entries_series[entry.Id] { + ser = append(ser, seriesmapid[series.Series()]) + } + + agents := []*dbmodels.Agent{} + for _, agent := range r_entries_agents[entry.Id] { + agents = append(agents, agentsmapid[agent.Agent()]) + } + + if err = dbmodels.BulkInsertFTS5Entry( + qe, + entry, + places, + agents, + ser, + ); err != nil { + app.Logger().Error("Error inserting entry", "error", err, "entry", entry) + } + } + + for _, content := range contentsmap { + agents := []*dbmodels.Agent{} + for _, agent := range r_contents_agents[content.Id] { + agents = append(agents, agentsmapid[agent.Agent()]) + } + + entry := entriesmapid[content.Entry()] + + if err = dbmodels.BulkInsertFTS5Content( + qc, + content, + entry, + agents, + ); err != nil { + app.Logger().Error("Error inserting content", "error", err, "content", content) + } + } + return nil }, func(app core.App) error { - return delete_data(app) + return errors.Join( + delete_data(app), + dbmodels.DeleteFTS5Data(app), + ) }) } func delete_data(app core.App) error { - _ = deleteTableContents(app, dbmodels.RelationTableName(dbmodels.ENTRIES_TABLE, dbmodels.SERIES_TABLE)) - _ = deleteTableContents(app, dbmodels.RelationTableName(dbmodels.ENTRIES_TABLE, dbmodels.AGENTS_TABLE)) - _ = deleteTableContents(app, dbmodels.RelationTableName(dbmodels.CONTENTS_TABLE, dbmodels.AGENTS_TABLE)) - _ = deleteTableContents(app, dbmodels.CONTENTS_TABLE) - _ = deleteTableContents(app, dbmodels.ENTRIES_TABLE) - _ = deleteTableContents(app, dbmodels.SERIES_TABLE) - _ = deleteTableContents(app, dbmodels.AGENTS_TABLE) - _ = deleteTableContents(app, dbmodels.PLACES_TABLE) - return nil + err1 := deleteTableContents(app, dbmodels.RelationTableName(dbmodels.ENTRIES_TABLE, dbmodels.SERIES_TABLE)) + err2 := deleteTableContents(app, dbmodels.RelationTableName(dbmodels.ENTRIES_TABLE, dbmodels.AGENTS_TABLE)) + err3 := deleteTableContents(app, dbmodels.RelationTableName(dbmodels.CONTENTS_TABLE, dbmodels.AGENTS_TABLE)) + err4 := deleteTableContents(app, dbmodels.CONTENTS_TABLE) + err5 := deleteTableContents(app, dbmodels.ENTRIES_TABLE) + err6 := deleteTableContents(app, dbmodels.SERIES_TABLE) + err7 := deleteTableContents(app, dbmodels.ITEMS_TABLE) + err8 := deleteTableContents(app, dbmodels.AGENTS_TABLE) + err9 := deleteTableContents(app, dbmodels.PLACES_TABLE) + return errors.Join(err1, err2, err3, err4, err5, err6, err7, err8, err9) } func deleteTableContents(app core.App, table string) error {