Theodor-Springmann-Stiftung/musenalm
1
0
Fork 0
mirror of https://github.com/Theodor-Springmann-Stiftung/musenalm.git synced 2025-10-29 09:15:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Speed up data insertion

Browse Source
This commit is contained in:
Simon Martens
2025-02-11 19:56:40 +01:00
parent 605f10d3c8
commit 0c8cd35577
5 changed files with 67 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
migrations/1739007272_insert_data.go
View File

@@ -2,6 +2,7 @@ package migrations
import ( import (
"github.com/Theodor-Springmann-Stiftung/musenalm/dbmodels" "github.com/Theodor-Springmann-Stiftung/musenalm/dbmodels"
"github.com/Theodor-Springmann-Stiftung/musenalm/helpers/datatypes"
"github.com/Theodor-Springmann-Stiftung/musenalm/migrations/seed" "github.com/Theodor-Springmann-Stiftung/musenalm/migrations/seed"
"github.com/Theodor-Springmann-Stiftung/musenalm/xmlmodels" "github.com/Theodor-Springmann-Stiftung/musenalm/xmlmodels"
"github.com/pocketbase/pocketbase/core" "github.com/pocketbase/pocketbase/core"
@@ -17,8 +18,9 @@ func init() {
adb.Reihen = xmlmodels.SanitizeReihen(adb.Reihen, adb.Relationen_Bände_Reihen) adb.Reihen = xmlmodels.SanitizeReihen(adb.Reihen, adb.Relationen_Bände_Reihen)
if records, err := seed.RecordsFromAkteure(app, adb.Akteure); err == nil { agents, err := seed.RecordsFromAkteure(app, adb.Akteure)
for _, record := range records { if err == nil {
for _, record := range agents {
if err = app.Save(record); err != nil { if err = app.Save(record); err != nil {
app.Logger().Error("Error saving record", "error", err, "record", record) app.Logger().Error("Error saving record", "error", err, "record", record)
} }
@@ -27,8 +29,9 @@ func init() {
panic(err) panic(err)
} }
if records, err := seed.RecordsFromOrte(app, adb.Orte); err == nil { places, err := seed.RecordsFromOrte(app, adb.Orte)
for _, record := range records { if err == nil {
for _, record := range places {
if err = app.Save(record); err != nil { if err = app.Save(record); err != nil {
app.Logger().Error("Error saving record", "error", err, "record", record) app.Logger().Error("Error saving record", "error", err, "record", record)
} }
@@ -37,8 +40,9 @@ func init() {
panic(err) panic(err)
} }
if records, err := seed.RecordsFromReihentitel(app, adb.Reihen); err == nil { series, err := seed.RecordsFromReihentitel(app, adb.Reihen)
for _, record := range records { if err == nil {
for _, record := range series {
if err = app.Save(record); err != nil { if err = app.Save(record); err != nil {
app.Logger().Error("Error saving record", "error", err, "record", record) app.Logger().Error("Error saving record", "error", err, "record", record)
} }
@@ -47,8 +51,9 @@ func init() {
panic(err) panic(err)
} }
if records, err := seed.RecordsFromBände(app, *adb); err == nil { entries, err := seed.RecordsFromBände(app, *adb)
for _, record := range records { if err == nil {
for _, record := range entries {
if err = app.Save(record); err != nil { if err = app.Save(record); err != nil {
app.Logger().Error("Error saving record", "error", err, "record", record) app.Logger().Error("Error saving record", "error", err, "record", record)
} }
@@ -67,8 +72,11 @@ func init() {
panic(err) panic(err)
} }
if records, err := seed.RecordsFromInhalte(app, adb.Inhalte); err == nil { entriesmap := datatypes.MakeMap(entries, func(record *dbmodels.Entry) string { return record.MusenalmID() })
for _, record := range records {
contents, err := seed.RecordsFromInhalte(app, adb.Inhalte, entriesmap)
if err == nil {
for _, record := range contents {
if err = app.Save(record); err != nil { if err = app.Save(record); err != nil {
app.Logger().Error("Error saving record", "error", err, "record", record) app.Logger().Error("Error saving record", "error", err, "record", record)
} }
@@ -77,7 +85,11 @@ func init() {
panic(err) panic(err)
} }
if records, err := seed.RecordsFromRelationBändeReihen(app, adb.Relationen_Bände_Reihen); err == nil { seriesmap := datatypes.MakeMap(series, func(record *dbmodels.Series) string { return record.MusenalmID() })
agentsmap := datatypes.MakeMap(agents, func(record *dbmodels.Agent) string { return record.MusenalmID() })
contentsmap := datatypes.MakeMap(contents, func(record *dbmodels.Content) string { return record.MusenalmID() })
if records, err := seed.RecordsFromRelationBändeReihen(app, adb.Relationen_Bände_Reihen, seriesmap, entriesmap); err == nil {
for _, record := range records { for _, record := range records {
if err := app.Save(record); err != nil { if err := app.Save(record); err != nil {
app.Logger().Error("Error saving record", "error", err, "record", record) app.Logger().Error("Error saving record", "error", err, "record", record)
@@ -87,7 +99,7 @@ func init() {
panic(err) panic(err)
} }
if records, err := seed.RecordsFromRelationBändeAkteure(app, adb.Relationen_Bände_Akteure); err == nil { if records, err := seed.RecordsFromRelationBändeAkteure(app, adb.Relationen_Bände_Akteure, entriesmap, agentsmap); err == nil {
for _, record := range records { for _, record := range records {
if err := app.Save(record); err != nil { if err := app.Save(record); err != nil {
app.Logger().Error("Error saving record", "error", err, "record", record) app.Logger().Error("Error saving record", "error", err, "record", record)
@@ -97,7 +109,7 @@ func init() {
panic(err) panic(err)
} }
if records, err := seed.RecordsFromRelationInhalteAkteure(app, adb.Relationen_Inhalte_Akteure); err == nil { if records, err := seed.RecordsFromRelationInhalteAkteure(app, adb.Relationen_Inhalte_Akteure, contentsmap, agentsmap); err == nil {
for _, record := range records { for _, record := range records {
if err := app.Save(record); err != nil { if err := app.Save(record); err != nil {
app.Logger().Error("Error saving record", "error", err, "record", record) app.Logger().Error("Error saving record", "error", err, "record", record)

12
migrations/seed/contents.go
View File

@@ -16,7 +16,11 @@ import (
const NO_TITLE = "[No Title]" const NO_TITLE = "[No Title]"
func RecordsFromInhalte(app core.App, inhalte xmlmodels.Inhalte) ([]*dbmodels.Content, error) { func RecordsFromInhalte(
app core.App,
inhalte xmlmodels.Inhalte,
entries map[string]*dbmodels.Entry,
) ([]*dbmodels.Content, error) {
collection, err := app.FindCollectionByNameOrId(dbmodels.CONTENTS_TABLE) collection, err := app.FindCollectionByNameOrId(dbmodels.CONTENTS_TABLE)
records := make([]*dbmodels.Content, 0, len(inhalte.Inhalte)) records := make([]*dbmodels.Content, 0, len(inhalte.Inhalte))
if err != nil { if err != nil {
@@ -29,11 +33,7 @@ func RecordsFromInhalte(app core.App, inhalte xmlmodels.Inhalte) ([]*dbmodels.Co
for i := 0; i < len(inhalte.Inhalte); i++ { for i := 0; i < len(inhalte.Inhalte); i++ {
record := dbmodels.NewContent(core.NewRecord(collection)) record := dbmodels.NewContent(core.NewRecord(collection))
inhalt := inhalte.Inhalte[i] inhalt := inhalte.Inhalte[i]
band, err := app.FindFirstRecordByData(dbmodels.ENTRIES_TABLE, dbmodels.MUSENALMID_FIELD, inhalt.Band) band, ok := entries[inhalt.Band]
if err != nil {
app.Logger().Error("Error finding band record for inhalt", "error", err, "inhalt", inhalt)
continue
}
record.SetEntry(band.Id) record.SetEntry(band.Id)
record.SetAnnotation(NormalizeString(inhalt.Anmerkungen)) record.SetAnnotation(NormalizeString(inhalt.Anmerkungen))

25
migrations/seed/r_contents_agents.go
View File

@@ -9,7 +9,12 @@ import (
"github.com/pocketbase/pocketbase/core" "github.com/pocketbase/pocketbase/core"
) )
func RecordsFromRelationInhalteAkteure(app core.App, relations xmlmodels.Relationen_Inhalte_Akteure) ([]*dbmodels.RContentsAgents, error) { func RecordsFromRelationInhalteAkteure(
app core.App,
relations xmlmodels.Relationen_Inhalte_Akteure,
contents map[string]*dbmodels.Content,
agents map[string]*dbmodels.Agent,
) ([]*dbmodels.RContentsAgents, error) {
records := make([]*dbmodels.RContentsAgents, 0, len(relations.Relationen)) records := make([]*dbmodels.RContentsAgents, 0, len(relations.Relationen))
collection, err := app.FindCollectionByNameOrId(dbmodels.RelationTableName(dbmodels.CONTENTS_TABLE, dbmodels.AGENTS_TABLE)) collection, err := app.FindCollectionByNameOrId(dbmodels.RelationTableName(dbmodels.CONTENTS_TABLE, dbmodels.AGENTS_TABLE))
if err != nil { if err != nil {
@@ -18,19 +23,17 @@ func RecordsFromRelationInhalteAkteure(app core.App, relations xmlmodels.Relatio
} }
for _, relation := range relations.Relationen { for _, relation := range relations.Relationen {
c, err := app.FindFirstRecordByData(dbmodels.CONTENTS_TABLE, dbmodels.MUSENALMID_FIELD, relation.Inhalt) content, ok := contents[relation.Inhalt]
if err != nil { if !ok {
app.Logger().Error("Error finding Inhalt", "error", err, "relation", relation)
continue
}
content := dbmodels.NewContent(c)
a, err := app.FindFirstRecordByData(dbmodels.AGENTS_TABLE, dbmodels.MUSENALMID_FIELD, relation.Akteur)
if err != nil {
app.Logger().Error("Error finding Content", "error", err, "relation", relation) app.Logger().Error("Error finding Content", "error", err, "relation", relation)
continue continue
} }
agent := dbmodels.NewAgent(a)
agent, ok := agents[relation.Akteur]
if !ok {
app.Logger().Error("Error finding Agent", "error", err, "relation", relation)
continue
}
record := dbmodels.NewRContentsAgents(core.NewRecord(collection)) record := dbmodels.NewRContentsAgents(core.NewRecord(collection))
record.SetContent(content.Id) record.SetContent(content.Id)

20
migrations/seed/r_entries_agents.go
View File

@@ -8,7 +8,12 @@ import (
"github.com/pocketbase/pocketbase/core" "github.com/pocketbase/pocketbase/core"
) )
func RecordsFromRelationBändeAkteure(app core.App, relations xmlmodels.Relationen_Bände_Akteure) ([]*dbmodels.REntriesAgents, error) { func RecordsFromRelationBändeAkteure(
app core.App,
relations xmlmodels.Relationen_Bände_Akteure,
entries map[string]*dbmodels.Entry,
agents map[string]*dbmodels.Agent,
) ([]*dbmodels.REntriesAgents, error) {
records := make([]*dbmodels.REntriesAgents, 0, len(relations.Relationen)) records := make([]*dbmodels.REntriesAgents, 0, len(relations.Relationen))
collection, err := app.FindCollectionByNameOrId(dbmodels.RelationTableName(dbmodels.ENTRIES_TABLE, dbmodels.AGENTS_TABLE)) collection, err := app.FindCollectionByNameOrId(dbmodels.RelationTableName(dbmodels.ENTRIES_TABLE, dbmodels.AGENTS_TABLE))
if err != nil { if err != nil {
@@ -17,14 +22,14 @@ func RecordsFromRelationBändeAkteure(app core.App, relations xmlmodels.Relation
} }
for _, relation := range relations.Relationen { for _, relation := range relations.Relationen {
entry, err := app.FindFirstRecordByData(dbmodels.ENTRIES_TABLE, dbmodels.MUSENALMID_FIELD, relation.Band) entry, ok := entries[relation.Band]
if err != nil { if !ok {
app.Logger().Error("Error finding Entry", "error", err, "relation", relation) app.Logger().Error("Error finding Entry", "error", err, "relation", relation)
continue continue
} }
agent, err := app.FindFirstRecordByData(dbmodels.AGENTS_TABLE, dbmodels.MUSENALMID_FIELD, relation.Akteur) agent, ok := agents[relation.Akteur]
if err != nil { if !ok {
app.Logger().Error("Error finding Agent", "error", err, "relation", relation) app.Logger().Error("Error finding Agent", "error", err, "relation", relation)
continue continue
} }
@@ -49,9 +54,8 @@ func RecordsFromRelationBändeAkteure(app core.App, relations xmlmodels.Relation
ser := record.Agent() ser := record.Agent()
if strings.TrimSpace(rel) == "" || strings.TrimSpace(ent) == "" || strings.TrimSpace(ser) == "" { if strings.TrimSpace(rel) == "" || strings.TrimSpace(ent) == "" || strings.TrimSpace(ser) == "" {
e := dbmodels.NewEntry(entry) entry.SetEditState(dbmodels.EDITORSTATE_VALUES[len(dbmodels.EDITORSTATE_VALUES)-2])
e.SetEditState(dbmodels.EDITORSTATE_VALUES[len(dbmodels.EDITORSTATE_VALUES)-2]) _ = app.Save(entry)
_ = app.Save(e)
} }
records = append(records, record) records = append(records, record)
} }

17
migrations/seed/r_entries_series.go
View File

@@ -8,7 +8,12 @@ import (
"github.com/pocketbase/pocketbase/core" "github.com/pocketbase/pocketbase/core"
) )
func RecordsFromRelationBändeReihen(app core.App, relations xmlmodels.Relationen_Bände_Reihen) ([]*dbmodels.REntriesSeries, error) { func RecordsFromRelationBändeReihen(
app core.App,
relations xmlmodels.Relationen_Bände_Reihen,
series map[string]*dbmodels.Series,
entries map[string]*dbmodels.Entry,
) ([]*dbmodels.REntriesSeries, error) {
records := make([]*dbmodels.REntriesSeries, 0, len(relations.Relationen)) records := make([]*dbmodels.REntriesSeries, 0, len(relations.Relationen))
collection, err := app.FindCollectionByNameOrId(dbmodels.RelationTableName(dbmodels.ENTRIES_TABLE, dbmodels.SERIES_TABLE)) collection, err := app.FindCollectionByNameOrId(dbmodels.RelationTableName(dbmodels.ENTRIES_TABLE, dbmodels.SERIES_TABLE))
if err != nil { if err != nil {
@@ -17,16 +22,14 @@ func RecordsFromRelationBändeReihen(app core.App, relations xmlmodels.Relatione
} }
for _, relation := range relations.Relationen { for _, relation := range relations.Relationen {
e, err := app.FindFirstRecordByData(dbmodels.ENTRIES_TABLE, dbmodels.MUSENALMID_FIELD, relation.Band) entry, ok := entries[relation.Band]
if err != nil { if !ok {
app.Logger().Error("Error finding Entry", "error", err, "relation", relation) app.Logger().Error("Error finding Entry", "error", err, "relation", relation)
continue continue
} }
entry := dbmodels.NewEntry(e) series, ok := series[relation.Reihe]
if !ok {
series, err := app.FindFirstRecordByData(dbmodels.SERIES_TABLE, dbmodels.MUSENALMID_FIELD, relation.Reihe)
if err != nil {
app.Logger().Error("Error finding Series", "error", err, "relation", relation) app.Logger().Error("Error finding Series", "error", err, "relation", relation)
continue continue
} }