From 0c8cd35577296eaa6fce655b5544ae2a56818dd5 Mon Sep 17 00:00:00 2001 From: Simon Martens Date: Tue, 11 Feb 2025 19:56:40 +0100 Subject: [PATCH] Speed up data insertion --- migrations/1739007272_insert_data.go | 38 ++++++++++++++++++---------- migrations/seed/contents.go | 12 ++++----- migrations/seed/r_contents_agents.go | 25 ++++++++++-------- migrations/seed/r_entries_agents.go | 20 +++++++++------ migrations/seed/r_entries_series.go | 17 ++++++++----- 5 files changed, 67 insertions(+), 45 deletions(-) diff --git a/migrations/1739007272_insert_data.go b/migrations/1739007272_insert_data.go index 0b9a1c6..098235f 100644 --- a/migrations/1739007272_insert_data.go +++ b/migrations/1739007272_insert_data.go @@ -2,6 +2,7 @@ package migrations import ( "github.com/Theodor-Springmann-Stiftung/musenalm/dbmodels" + "github.com/Theodor-Springmann-Stiftung/musenalm/helpers/datatypes" "github.com/Theodor-Springmann-Stiftung/musenalm/migrations/seed" "github.com/Theodor-Springmann-Stiftung/musenalm/xmlmodels" "github.com/pocketbase/pocketbase/core" @@ -17,8 +18,9 @@ func init() { adb.Reihen = xmlmodels.SanitizeReihen(adb.Reihen, adb.Relationen_Bände_Reihen) - if records, err := seed.RecordsFromAkteure(app, adb.Akteure); err == nil { - for _, record := range records { + agents, err := seed.RecordsFromAkteure(app, adb.Akteure) + if err == nil { + for _, record := range agents { if err = app.Save(record); err != nil { app.Logger().Error("Error saving record", "error", err, "record", record) } @@ -27,8 +29,9 @@ func init() { panic(err) } - if records, err := seed.RecordsFromOrte(app, adb.Orte); err == nil { - for _, record := range records { + places, err := seed.RecordsFromOrte(app, adb.Orte) + if err == nil { + for _, record := range places { if err = app.Save(record); err != nil { app.Logger().Error("Error saving record", "error", err, "record", record) } @@ -37,8 +40,9 @@ func init() { panic(err) } - if records, err := seed.RecordsFromReihentitel(app, adb.Reihen); err == nil { - for _, record := range records { + series, err := seed.RecordsFromReihentitel(app, adb.Reihen) + if err == nil { + for _, record := range series { if err = app.Save(record); err != nil { app.Logger().Error("Error saving record", "error", err, "record", record) } @@ -47,8 +51,9 @@ func init() { panic(err) } - if records, err := seed.RecordsFromBände(app, *adb); err == nil { - for _, record := range records { + entries, err := seed.RecordsFromBände(app, *adb) + if err == nil { + for _, record := range entries { if err = app.Save(record); err != nil { app.Logger().Error("Error saving record", "error", err, "record", record) } @@ -67,8 +72,11 @@ func init() { panic(err) } - if records, err := seed.RecordsFromInhalte(app, adb.Inhalte); err == nil { - for _, record := range records { + entriesmap := datatypes.MakeMap(entries, func(record *dbmodels.Entry) string { return record.MusenalmID() }) + + contents, err := seed.RecordsFromInhalte(app, adb.Inhalte, entriesmap) + if err == nil { + for _, record := range contents { if err = app.Save(record); err != nil { app.Logger().Error("Error saving record", "error", err, "record", record) } @@ -77,7 +85,11 @@ func init() { panic(err) } - if records, err := seed.RecordsFromRelationBändeReihen(app, adb.Relationen_Bände_Reihen); err == nil { + seriesmap := datatypes.MakeMap(series, func(record *dbmodels.Series) string { return record.MusenalmID() }) + agentsmap := datatypes.MakeMap(agents, func(record *dbmodels.Agent) string { return record.MusenalmID() }) + contentsmap := datatypes.MakeMap(contents, func(record *dbmodels.Content) string { return record.MusenalmID() }) + + if records, err := seed.RecordsFromRelationBändeReihen(app, adb.Relationen_Bände_Reihen, seriesmap, entriesmap); err == nil { for _, record := range records { if err := app.Save(record); err != nil { app.Logger().Error("Error saving record", "error", err, "record", record) @@ -87,7 +99,7 @@ func init() { panic(err) } - if records, err := seed.RecordsFromRelationBändeAkteure(app, adb.Relationen_Bände_Akteure); err == nil { + if records, err := seed.RecordsFromRelationBändeAkteure(app, adb.Relationen_Bände_Akteure, entriesmap, agentsmap); err == nil { for _, record := range records { if err := app.Save(record); err != nil { app.Logger().Error("Error saving record", "error", err, "record", record) @@ -97,7 +109,7 @@ func init() { panic(err) } - if records, err := seed.RecordsFromRelationInhalteAkteure(app, adb.Relationen_Inhalte_Akteure); err == nil { + if records, err := seed.RecordsFromRelationInhalteAkteure(app, adb.Relationen_Inhalte_Akteure, contentsmap, agentsmap); err == nil { for _, record := range records { if err := app.Save(record); err != nil { app.Logger().Error("Error saving record", "error", err, "record", record) diff --git a/migrations/seed/contents.go b/migrations/seed/contents.go index 18b7851..e564ce3 100644 --- a/migrations/seed/contents.go +++ b/migrations/seed/contents.go @@ -16,7 +16,11 @@ import ( const NO_TITLE = "[No Title]" -func RecordsFromInhalte(app core.App, inhalte xmlmodels.Inhalte) ([]*dbmodels.Content, error) { +func RecordsFromInhalte( + app core.App, + inhalte xmlmodels.Inhalte, + entries map[string]*dbmodels.Entry, +) ([]*dbmodels.Content, error) { collection, err := app.FindCollectionByNameOrId(dbmodels.CONTENTS_TABLE) records := make([]*dbmodels.Content, 0, len(inhalte.Inhalte)) if err != nil { @@ -29,11 +33,7 @@ func RecordsFromInhalte(app core.App, inhalte xmlmodels.Inhalte) ([]*dbmodels.Co for i := 0; i < len(inhalte.Inhalte); i++ { record := dbmodels.NewContent(core.NewRecord(collection)) inhalt := inhalte.Inhalte[i] - band, err := app.FindFirstRecordByData(dbmodels.ENTRIES_TABLE, dbmodels.MUSENALMID_FIELD, inhalt.Band) - if err != nil { - app.Logger().Error("Error finding band record for inhalt", "error", err, "inhalt", inhalt) - continue - } + band, ok := entries[inhalt.Band] record.SetEntry(band.Id) record.SetAnnotation(NormalizeString(inhalt.Anmerkungen)) diff --git a/migrations/seed/r_contents_agents.go b/migrations/seed/r_contents_agents.go index 421a920..35f7e2c 100644 --- a/migrations/seed/r_contents_agents.go +++ b/migrations/seed/r_contents_agents.go @@ -9,7 +9,12 @@ import ( "github.com/pocketbase/pocketbase/core" ) -func RecordsFromRelationInhalteAkteure(app core.App, relations xmlmodels.Relationen_Inhalte_Akteure) ([]*dbmodels.RContentsAgents, error) { +func RecordsFromRelationInhalteAkteure( + app core.App, + relations xmlmodels.Relationen_Inhalte_Akteure, + contents map[string]*dbmodels.Content, + agents map[string]*dbmodels.Agent, +) ([]*dbmodels.RContentsAgents, error) { records := make([]*dbmodels.RContentsAgents, 0, len(relations.Relationen)) collection, err := app.FindCollectionByNameOrId(dbmodels.RelationTableName(dbmodels.CONTENTS_TABLE, dbmodels.AGENTS_TABLE)) if err != nil { @@ -18,19 +23,17 @@ func RecordsFromRelationInhalteAkteure(app core.App, relations xmlmodels.Relatio } for _, relation := range relations.Relationen { - c, err := app.FindFirstRecordByData(dbmodels.CONTENTS_TABLE, dbmodels.MUSENALMID_FIELD, relation.Inhalt) - if err != nil { - app.Logger().Error("Error finding Inhalt", "error", err, "relation", relation) - continue - } - content := dbmodels.NewContent(c) - - a, err := app.FindFirstRecordByData(dbmodels.AGENTS_TABLE, dbmodels.MUSENALMID_FIELD, relation.Akteur) - if err != nil { + content, ok := contents[relation.Inhalt] + if !ok { app.Logger().Error("Error finding Content", "error", err, "relation", relation) continue } - agent := dbmodels.NewAgent(a) + + agent, ok := agents[relation.Akteur] + if !ok { + app.Logger().Error("Error finding Agent", "error", err, "relation", relation) + continue + } record := dbmodels.NewRContentsAgents(core.NewRecord(collection)) record.SetContent(content.Id) diff --git a/migrations/seed/r_entries_agents.go b/migrations/seed/r_entries_agents.go index c9be477..b7e24be 100644 --- a/migrations/seed/r_entries_agents.go +++ b/migrations/seed/r_entries_agents.go @@ -8,7 +8,12 @@ import ( "github.com/pocketbase/pocketbase/core" ) -func RecordsFromRelationBändeAkteure(app core.App, relations xmlmodels.Relationen_Bände_Akteure) ([]*dbmodels.REntriesAgents, error) { +func RecordsFromRelationBändeAkteure( + app core.App, + relations xmlmodels.Relationen_Bände_Akteure, + entries map[string]*dbmodels.Entry, + agents map[string]*dbmodels.Agent, +) ([]*dbmodels.REntriesAgents, error) { records := make([]*dbmodels.REntriesAgents, 0, len(relations.Relationen)) collection, err := app.FindCollectionByNameOrId(dbmodels.RelationTableName(dbmodels.ENTRIES_TABLE, dbmodels.AGENTS_TABLE)) if err != nil { @@ -17,14 +22,14 @@ func RecordsFromRelationBändeAkteure(app core.App, relations xmlmodels.Relation } for _, relation := range relations.Relationen { - entry, err := app.FindFirstRecordByData(dbmodels.ENTRIES_TABLE, dbmodels.MUSENALMID_FIELD, relation.Band) - if err != nil { + entry, ok := entries[relation.Band] + if !ok { app.Logger().Error("Error finding Entry", "error", err, "relation", relation) continue } - agent, err := app.FindFirstRecordByData(dbmodels.AGENTS_TABLE, dbmodels.MUSENALMID_FIELD, relation.Akteur) - if err != nil { + agent, ok := agents[relation.Akteur] + if !ok { app.Logger().Error("Error finding Agent", "error", err, "relation", relation) continue } @@ -49,9 +54,8 @@ func RecordsFromRelationBändeAkteure(app core.App, relations xmlmodels.Relation ser := record.Agent() if strings.TrimSpace(rel) == "" || strings.TrimSpace(ent) == "" || strings.TrimSpace(ser) == "" { - e := dbmodels.NewEntry(entry) - e.SetEditState(dbmodels.EDITORSTATE_VALUES[len(dbmodels.EDITORSTATE_VALUES)-2]) - _ = app.Save(e) + entry.SetEditState(dbmodels.EDITORSTATE_VALUES[len(dbmodels.EDITORSTATE_VALUES)-2]) + _ = app.Save(entry) } records = append(records, record) } diff --git a/migrations/seed/r_entries_series.go b/migrations/seed/r_entries_series.go index 78f234b..77cac6b 100644 --- a/migrations/seed/r_entries_series.go +++ b/migrations/seed/r_entries_series.go @@ -8,7 +8,12 @@ import ( "github.com/pocketbase/pocketbase/core" ) -func RecordsFromRelationBändeReihen(app core.App, relations xmlmodels.Relationen_Bände_Reihen) ([]*dbmodels.REntriesSeries, error) { +func RecordsFromRelationBändeReihen( + app core.App, + relations xmlmodels.Relationen_Bände_Reihen, + series map[string]*dbmodels.Series, + entries map[string]*dbmodels.Entry, +) ([]*dbmodels.REntriesSeries, error) { records := make([]*dbmodels.REntriesSeries, 0, len(relations.Relationen)) collection, err := app.FindCollectionByNameOrId(dbmodels.RelationTableName(dbmodels.ENTRIES_TABLE, dbmodels.SERIES_TABLE)) if err != nil { @@ -17,16 +22,14 @@ func RecordsFromRelationBändeReihen(app core.App, relations xmlmodels.Relatione } for _, relation := range relations.Relationen { - e, err := app.FindFirstRecordByData(dbmodels.ENTRIES_TABLE, dbmodels.MUSENALMID_FIELD, relation.Band) - if err != nil { + entry, ok := entries[relation.Band] + if !ok { app.Logger().Error("Error finding Entry", "error", err, "relation", relation) continue } - entry := dbmodels.NewEntry(e) - - series, err := app.FindFirstRecordByData(dbmodels.SERIES_TABLE, dbmodels.MUSENALMID_FIELD, relation.Reihe) - if err != nil { + series, ok := series[relation.Reihe] + if !ok { app.Logger().Error("Error finding Series", "error", err, "relation", relation) continue }