From bc244fbad4b2ce0da25a9082a025288c2728031f Mon Sep 17 00:00:00 2001 From: Simon Martens Date: Fri, 22 Nov 2024 00:35:27 +0100 Subject: [PATCH] XML parsing overhaul --- app/kgpz.go | 3 +- controllers/issue.go | 2 +- functions/date.go | 66 ++++++++++ go.mod | 2 +- go.sum | 2 + kgpz_web.go | 6 +- providers/gnd/gnd.go | 9 +- providers/gnd/model.go | 82 ++++++------- providers/xmlprovider/agents.go | 29 +---- providers/xmlprovider/categories.go | 29 +---- providers/xmlprovider/issues.go | 34 ++---- providers/xmlprovider/pieces.go | 37 ++---- providers/xmlprovider/places.go | 29 +---- providers/xmlprovider/roots.go | 111 +++++++++++++++++ providers/xmlprovider/works.go | 29 +---- providers/xmlprovider/xmlcommon.go | 4 + providers/xmlprovider/xmlprovider.go | 175 ++++++++++++++++++--------- server/server.go | 17 +-- templating/context.go | 16 ++- templating/engine.go | 58 +++++++-- templating/layout_registry.go | 14 ++- templating/template_registry.go | 12 +- viewmodels/issue.go | 60 ++------- viewmodels/singleissue.go | 9 +- viewmodels/year.go | 15 +-- views/routes/body.tmpl | 9 +- 26 files changed, 507 insertions(+), 352 deletions(-) create mode 100644 functions/date.go create mode 100644 providers/xmlprovider/roots.go diff --git a/app/kgpz.go b/app/kgpz.go index 68f5c35..d73e967 100644 --- a/app/kgpz.go +++ b/app/kgpz.go @@ -85,8 +85,7 @@ func (k *KGPZ) Enrich() error { } // INFO: We pass agents by value since we don't want to block the library - agents := make([]xmlprovider.Agent, len(k.Library.Agents.Items.Agents)) - _ = copy(agents, k.Library.Agents.Items.Agents) + agents := k.Library.Agents.All() go func(agents []xmlprovider.Agent) { k.GND.FetchPersons(agents) k.GND.WriteCache(k.Config.GNDPath) diff --git a/controllers/issue.go b/controllers/issue.go index 4b6b45b..494ce69 100644 --- a/controllers/issue.go +++ b/controllers/issue.go @@ -21,7 +21,7 @@ func GetIssue(kgpz *app.KGPZ) fiber.Handler { return c.SendStatus(fiber.StatusNotFound) } - issue, err := viewmodels.IssueView(y, d, kgpz.Library) + issue, err := viewmodels.NewSingleIssueView(y, d, kgpz.Library) if err != nil { logging.Error(err, "Issue could not be found") diff --git a/functions/date.go b/functions/date.go new file mode 100644 index 0000000..b2fee93 --- /dev/null +++ b/functions/date.go @@ -0,0 +1,66 @@ +package functions + +import ( + "time" +) + +const TLAYOUT = "2006-01-02" + +var TRANSLM = [][]string{ + {"Januar", "Jan", "1"}, + {"Februar", "Feb", "2"}, + {"März", "Mär", "3"}, + {"April", "Apr", "4"}, + {"Mai", "Mai", "5"}, + {"Juni", "Jun", "6"}, + {"Juli", "Jul", "7"}, + {"August", "Aug", "8"}, + {"September", "Sep", "9"}, + {"Oktober", "Okt", "10"}, + {"November", "Nov", "11"}, + {"Dezember", "Dez", "12"}, +} + +var TRANSLD = [][]string{ + {"Montag", "Mo"}, + {"Dienstag", "Di"}, + {"Mittwoch", "Mi"}, + {"Donnerstag", "Do"}, + {"Freitag", "Fr"}, + {"Samstag", "Sa"}, + {"Sonntag", "So"}, +} + +type Date struct { + Month string + Mon string + MonthNo string + DayNo int + Weekday string + Wd string +} + +func GetDate(d string) Date { + t, err := time.Parse(TLAYOUT, d) + if err != nil { + return Date{} + } + m := int(t.Month()) - 1 + wd := int(t.Weekday()) - 1 + return Date{ + Month: TRANSLM[m][0], + Mon: TRANSLM[m][1], + MonthNo: TRANSLM[m][2], + DayNo: t.Day(), + Weekday: TRANSLD[wd][0], + Wd: TRANSLD[wd][1], + } +} + +func MonthName(m int) string { + return TRANSLM[m-1][0] +} + +func MonthNameShort(m int) string { + return TRANSLM[m-1][1] +} diff --git a/go.mod b/go.mod index c317431..77d96a9 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/google/uuid v1.5.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.17.0 // indirect diff --git a/go.sum b/go.sum index 62d90ca..fe2bfee 100644 --- a/go.sum +++ b/go.sum @@ -46,6 +46,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= diff --git a/kgpz_web.go b/kgpz_web.go index 5560def..67f58da 100644 --- a/kgpz_web.go +++ b/kgpz_web.go @@ -11,6 +11,8 @@ import ( "github.com/Theodor-Springmann-Stiftung/kgpz_web/helpers/logging" "github.com/Theodor-Springmann-Stiftung/kgpz_web/providers" "github.com/Theodor-Springmann-Stiftung/kgpz_web/server" + "github.com/Theodor-Springmann-Stiftung/kgpz_web/templating" + "github.com/Theodor-Springmann-Stiftung/kgpz_web/views" ) const ( @@ -44,7 +46,9 @@ func main() { kgpz := app.NewKGPZ(cfg) kgpz.Init() - server := server.Create(kgpz, cfg) + engine := templating.NewEngine(&views.LayoutFS, &views.RoutesFS, kgpz) + + server := server.Create(kgpz, cfg, engine) Start(kgpz, server) } diff --git a/providers/gnd/gnd.go b/providers/gnd/gnd.go index 51ad668..fb581d2 100644 --- a/providers/gnd/gnd.go +++ b/providers/gnd/gnd.go @@ -25,6 +25,8 @@ type GNDProvider struct { mu sync.Mutex Persons sync.Map + // INFO: this holds all errors that occured during fetching + // and is used to prevent further fetches of the same person. errmu sync.Mutex errs map[string]int } @@ -98,7 +100,6 @@ func (p *GNDProvider) readPerson(file string) { p.Persons.Store(person.Agent.GND, person) return } - } func (p *GNDProvider) WriteCache(folder string) error { @@ -134,6 +135,7 @@ func (p *GNDProvider) writePersons(folder string) error { return nil } +// INFO: this overwrites any existing files func (p *GNDProvider) writePerson(folder, id string, person Person) { // JSON marshalling of the person and sanity check: filepath := filepath.Join(folder, person.KGPZID+".json") @@ -209,9 +211,10 @@ func (p *GNDProvider) fetchPerson(person xmlprovider.Agent) { var response *http.Response + // INFO: we do 3 retries with increasing time between them for i := 0; i < 3; i++ { response, err = http.DefaultClient.Do(request) - if err == nil && 400 > response.StatusCode { + if err == nil && response.StatusCode < 400 { if i > 0 { logging.Info("Successfully fetched person: " + person.ID + " after " + strconv.Itoa(i) + " retries") } @@ -245,7 +248,7 @@ func (p *GNDProvider) fetchPerson(person xmlprovider.Agent) { return } - // Wirte response body to file: + // For debug purposes: Write response body to file: // os.WriteFile("gnd_responses/"+person.ID+".json", body, 0644) gndPerson := Person{} diff --git a/providers/gnd/model.go b/providers/gnd/model.go index 6e89bda..62a5698 100644 --- a/providers/gnd/model.go +++ b/providers/gnd/model.go @@ -9,65 +9,65 @@ import ( type Person struct { KGPZID string `json:"kgpzid"` Agent xmlprovider.Agent `json:"agent"` - URL string `json:"id"` - DateOfBirth []string `json:"dateOfBirth"` - PlaceOfBirth []Entity `json:"placeOfBirth"` - DateOfDeath []string `json:"dateOfDeath"` - PlaceOfDeath []Entity `json:"placeOfDeath"` - PlaceOfBirthAsLiteral []string `json:"placeOfBirthAsLiteral"` - PlaceOfDeathAsLiteral []string `json:"placeOfDeathAsLiteral"` - BiographicalOrHistoricalInformation []string `json:"biographicalOrHistoricalInformation"` - PreferredName string `json:"preferredName"` - GndIdentifier string `json:"gndIdentifier"` - Wikipedia []Entity `json:"wikipedia"` - Depiction []Picture `json:"depiction"` - ProfessionOrOccupation []Entity `json:"professionOrOccupation"` - PreferredNameEntityForThePerson PersonNameEntity `json:"preferredNameEntityForThePerson"` - VariantNameEntityForThePerson []PersonNameEntity `json:"variantNameEntityForThePerson"` - VariantName []string `json:"variantName"` - SameAs []CrossReferences `json:"sameAs"` - Pseudonym []Entity `json:"pseudonym"` - GNDSubjectCategory []Entity `json:"gndSubjectCategory"` - Type []string `json:"type"` - PlaceOfActivity []Entity `json:"placeOfActivity"` + URL string `json:"id,omitempty"` + DateOfBirth []string `json:"dateOfBirth,omitempty"` + PlaceOfBirth []Entity `json:"placeOfBirth,omitempty"` + DateOfDeath []string `json:"dateOfDeath,omitempty"` + PlaceOfDeath []Entity `json:"placeOfDeath,omitempty"` + PlaceOfBirthAsLiteral []string `json:"placeOfBirthAsLiteral,omitempty"` + PlaceOfDeathAsLiteral []string `json:"placeOfDeathAsLiteral,omitempty"` + BiographicalOrHistoricalInformation []string `json:"biographicalOrHistoricalInformation,omitempty"` + PreferredName string `json:"preferredName,omitempty"` + GndIdentifier string `json:"gndIdentifier,omitempty"` + Wikipedia []Entity `json:"wikipedia,omitempty"` + Depiction []Picture `json:"depiction,omitempty"` + ProfessionOrOccupation []Entity `json:"professionOrOccupation,omitempty"` + PreferredNameEntityForThePerson PersonNameEntity `json:"preferredNameEntityForThePerson,omitempty"` + VariantNameEntityForThePerson []PersonNameEntity `json:"variantNameEntityForThePerson,omitempty"` + VariantName []string `json:"variantName,omitempty"` + SameAs []CrossReferences `json:"sameAs,omitempty"` + Pseudonym []Entity `json:"pseudonym,omitempty"` + GNDSubjectCategory []Entity `json:"gndSubjectCategory,omitempty"` + Type []string `json:"type,omitempty"` + PlaceOfActivity []Entity `json:"placeOfActivity,omitempty"` } type CrossReferences struct { - Items Collection `json:"collection"` - ID string `json:"id"` + Items Collection `json:"collection,omitempty"` + ID string `json:"id,omitempty"` } type Collection struct { - Abbr string `json:"abbr"` - Name string `json:"name"` - Publisher string `json:"publisher"` - Icon string `json:"icon"` - ID string `json:"id"` + Abbr string `json:"abbr,omitempty"` + Name string `json:"name,omitempty"` + Publisher string `json:"publisher,omitempty"` + Icon string `json:"icon,omitempty"` + ID string `json:"id,omitempty"` } type Link struct { - ID string `json:"id"` - Label string `json:"label"` + ID string `json:"id,omitempty"` + Label string `json:"label,omitempty"` } type Picture struct { - ID string `json:"id"` - URL string `json:"url"` - Thumbnail string `json:"thumbnail"` + ID string `json:"id,omitempty"` + URL string `json:"url,omitempty"` + Thumbnail string `json:"thumbnail,omitempty"` } type Entity struct { - ID string `json:"id"` - Label string `json:"label"` + ID string `json:"id,omitempty"` + Label string `json:"label,omitempty"` } type PersonNameEntity struct { - Prefix []string `json:"prefix"` - Counting []string `json:"counting"` - Forename []string `json:"forename"` - Surname []string `json:"surname"` - PersonalName []string `json:"personalName"` - NameAddition []string `json:"nameAddition"` + Prefix []string `json:"prefix,omitempty"` + Counting []string `json:"counting,omitempty"` + Forename []string `json:"forename,omitempty"` + Surname []string `json:"surname,omitempty"` + PersonalName []string `json:"personalName,omitempty"` + NameAddition []string `json:"nameAddition,omitempty"` } func (p Person) String() string { diff --git a/providers/xmlprovider/agents.go b/providers/xmlprovider/agents.go index 1a2a6c0..2e30119 100644 --- a/providers/xmlprovider/agents.go +++ b/providers/xmlprovider/agents.go @@ -5,10 +5,6 @@ import ( "fmt" ) -type AgentProvider struct { - XMLProvider[Agents] -} - type Agent struct { XMLName xml.Name `xml:"akteur"` Names []string `xml:"name"` @@ -20,29 +16,6 @@ type Agent struct { AnnotationNote } -type Agents struct { - XMLName xml.Name `xml:"akteure"` - Agents []Agent `xml:"akteur"` -} - -func (a Agents) String() string { - var res []string - for _, agent := range a.Agents { - res = append(res, agent.String()) - } - - return fmt.Sprintf("Agents: %v", res) -} - -func (a Agents) Append(data Agents) Agents { - a.Agents = append(a.Agents, data.Agents...) - return a -} - -func (a *Agent) String() string { +func (a Agent) String() string { return fmt.Sprintf("ID: %s\nNames: %v\nSortName: %s\nLife: %s\nGND: %s\nAnnotations: %v\nNotes: %v\n", a.ID, a.Names, a.SortName, a.Life, a.GND, a.Annotations, a.Notes) } - -func NewAgentProvider(paths []string) *AgentProvider { - return &AgentProvider{XMLProvider: XMLProvider[Agents]{paths: paths}} -} diff --git a/providers/xmlprovider/categories.go b/providers/xmlprovider/categories.go index b55567f..c7611f5 100644 --- a/providers/xmlprovider/categories.go +++ b/providers/xmlprovider/categories.go @@ -5,15 +5,6 @@ import ( "fmt" ) -type CategoryProvider struct { - XMLProvider[Categories] -} - -type Categories struct { - XMLName xml.Name `xml:"kategorien"` - Category []Category `xml:"kategorie"` -} - type Category struct { XMLName xml.Name `xml:"kategorie"` Names []string `xml:"name"` @@ -22,24 +13,6 @@ type Category struct { AnnotationNote } -func (c Categories) Append(data Categories) Categories { - c.Category = append(c.Category, data.Category...) - return c -} - -func (c Categories) String() string { - var res []string - for _, category := range c.Category { - res = append(res, category.String()) - } - - return fmt.Sprintf("Categories: %v", res) -} - -func (c *Category) String() string { +func (c Category) String() string { return fmt.Sprintf("ID: %s\nNames: %v\nSortName: %s\nAnnotations: %v\nNotes: %v\n", c.ID, c.Names, c.SortName, c.Annotations, c.Notes) } - -func NewCategoryProvider(paths []string) *CategoryProvider { - return &CategoryProvider{XMLProvider: XMLProvider[Categories]{paths: paths}} -} diff --git a/providers/xmlprovider/issues.go b/providers/xmlprovider/issues.go index 9fb5ef9..5d76140 100644 --- a/providers/xmlprovider/issues.go +++ b/providers/xmlprovider/issues.go @@ -3,17 +3,9 @@ package xmlprovider import ( "encoding/xml" "fmt" + "strconv" ) -type IssueProvider struct { - XMLProvider[Issues] -} - -type Issues struct { - XMLName xml.Name `xml:"stuecke"` - Issues []Issue `xml:"stueck"` -} - type Issue struct { XMLName xml.Name `xml:"stueck"` Number Nummer `xml:"nummer"` @@ -37,24 +29,20 @@ type Additional struct { Bis string `xml:"bis"` } -func (i Issues) Append(data Issues) Issues { - i.Issues = append(i.Issues, data.Issues...) - return i -} - -func (i Issues) String() string { - var res []string - for _, issue := range i.Issues { - res = append(res, issue.String()) +func (i Issue) GetIDs() []string { + res := make([]string, 2) + date := i.Datum.When + if date != "" { + res = append(res, date) } - return fmt.Sprintf("Issues: %v", res) + if len(date) > 4 { + res = append(res, i.Datum.When[0:4]+"-"+strconv.Itoa(i.Number.No)) + } + + return res } func (i Issue) String() string { return fmt.Sprintf("Number: %v, Datum: %v, Von: %d, Bis: %d, Additionals: %v, Identifier: %v, AnnotationNote: %v\n", i.Number, i.Datum, i.Von, i.Bis, i.Additionals, i.Identifier, i.AnnotationNote) } - -func NewIssueProvider(paths []string) *IssueProvider { - return &IssueProvider{XMLProvider: XMLProvider[Issues]{paths: paths}} -} diff --git a/providers/xmlprovider/pieces.go b/providers/xmlprovider/pieces.go index 1edfac1..c7067f4 100644 --- a/providers/xmlprovider/pieces.go +++ b/providers/xmlprovider/pieces.go @@ -3,17 +3,10 @@ package xmlprovider import ( "encoding/xml" "fmt" + + "github.com/google/uuid" ) -type PieceProvider struct { - XMLProvider[Pieces] -} - -type Pieces struct { - XMLName xml.Name `xml:"beitraege"` - Piece []Piece `xml:"beitrag"` -} - type Piece struct { XMLName xml.Name `xml:"beitrag"` IssueRefs []IssueRef `xml:"stueck"` @@ -30,24 +23,18 @@ type Piece struct { AnnotationNote } -func (p Pieces) Append(data Pieces) Pieces { - p.Piece = append(p.Piece, data.Piece...) - return p -} - -func (p Pieces) String() string { - var res []string - for _, piece := range p.Piece { - res = append(res, piece.String()) - } - - return fmt.Sprintf("Pieces: %v", res) -} - func (p Piece) String() string { return fmt.Sprintf("ID: %s\nIssueRefs: %v\nPlaceRefs: %v\nCategoryRefs: %v\nAgentRefs: %v\nWorkRefs: %v\nPieceRefs: %v\nAdditionalRef: %v\nIncipit: %v\nTitle: %v\nAnnotations: %v\nNotes: %v\n", p.ID, p.IssueRefs, p.PlaceRefs, p.CategoryRefs, p.AgentRefs, p.WorkRefs, p.PieceRefs, p.AdditionalRef, p.Incipit, p.Title, p.Annotations, p.Notes) } -func NewPieceProvider(paths []string) *PieceProvider { - return &PieceProvider{XMLProvider: XMLProvider[Pieces]{paths: paths}} +func (p Piece) GetIDs() []string { + ret := make([]string, 2) + if p.ID != "" { + ret = append(ret, p.ID) + } + + // TODO: sensible IDs + uid := uuid.New() + ret = append(ret, uid.String()) + return ret } diff --git a/providers/xmlprovider/places.go b/providers/xmlprovider/places.go index 38c2b2b..0d86488 100644 --- a/providers/xmlprovider/places.go +++ b/providers/xmlprovider/places.go @@ -5,15 +5,6 @@ import ( "fmt" ) -type PlaceProvider struct { - XMLProvider[Places] -} - -type Places struct { - XMLName xml.Name `xml:"orte"` - Place []Place `xml:"ort"` -} - type Place struct { XMLName xml.Name `xml:"ort"` Names []string `xml:"name"` @@ -23,24 +14,6 @@ type Place struct { AnnotationNote } -func (p Places) Append(data Places) Places { - p.Place = append(p.Place, data.Place...) - return p -} - -func (p Places) String() string { - var res []string - for _, place := range p.Place { - res = append(res, place.String()) - } - - return fmt.Sprintf("Places: %v", res) -} - -func (p *Place) String() string { +func (p Place) String() string { return fmt.Sprintf("ID: %s\nNames: %v\nSortName: %s\nGeo: %s\nAnnotations: %v\nNotes: %v\n", p.ID, p.Names, p.SortName, p.Geo, p.Annotations, p.Notes) } - -func NewPlaceProvider(paths []string) *PlaceProvider { - return &PlaceProvider{XMLProvider: XMLProvider[Places]{paths: paths}} -} diff --git a/providers/xmlprovider/roots.go b/providers/xmlprovider/roots.go new file mode 100644 index 0000000..eeec5af --- /dev/null +++ b/providers/xmlprovider/roots.go @@ -0,0 +1,111 @@ +package xmlprovider + +import "encoding/xml" + +// INFO: These are just root elements that hold the data of the XML files. +// They get discarded after a parse. +type XMLRootElement[T any] interface { + Children() []T +} + +type AgentRoot struct { + XMLName xml.Name `xml:"akteure"` + Agents []Agent `xml:"akteur"` +} + +func NewAgentRoot() *AgentRoot { + return &AgentRoot{} +} + +func (a AgentRoot) New() *AgentRoot { + return NewAgentRoot() +} + +func (a AgentRoot) Children() []Agent { + return a.Agents +} + +type PlaceRoot struct { + XMLName xml.Name `xml:"orte"` + Place []Place `xml:"ort"` +} + +func NewPlaceRoot() *PlaceRoot { + return &PlaceRoot{} +} + +func (p PlaceRoot) New() *PlaceRoot { + return NewPlaceRoot() +} + +func (p PlaceRoot) Children() []Place { + return p.Place +} + +type CategoryRoot struct { + XMLName xml.Name `xml:"kategorien"` + Category []Category `xml:"kategorie"` +} + +func NewCategoryRoot() *CategoryRoot { + return &CategoryRoot{} +} + +func (c CategoryRoot) New() XMLRootElement[Category] { + return NewCategoryRoot() +} + +func (c CategoryRoot) Children() []Category { + return c.Category +} + +type PieceRoot struct { + XMLName xml.Name `xml:"beitraege"` + Piece []Piece `xml:"beitrag"` +} + +func NewPieceRoot() *PieceRoot { + return &PieceRoot{} +} + +func (p PieceRoot) New() XMLRootElement[Piece] { + return NewPieceRoot() +} + +func (p PieceRoot) Children() []Piece { + return p.Piece +} + +type IssueRoot struct { + XMLName xml.Name `xml:"stuecke"` + Issues []Issue `xml:"stueck"` +} + +func NewIssueRoot() *IssueRoot { + return &IssueRoot{} +} + +func (i IssueRoot) New() XMLRootElement[Issue] { + return NewIssueRoot() +} + +func (i IssueRoot) Children() []Issue { + return i.Issues +} + +type WorkRoot struct { + XMLName xml.Name `xml:"werke"` + Work []Work `xml:"werk"` +} + +func NewWorkRoot() *WorkRoot { + return &WorkRoot{} +} + +func (w WorkRoot) New() XMLRootElement[Work] { + return NewWorkRoot() +} + +func (w WorkRoot) Children() []Work { + return w.Work +} diff --git a/providers/xmlprovider/works.go b/providers/xmlprovider/works.go index a9268f8..41ea442 100644 --- a/providers/xmlprovider/works.go +++ b/providers/xmlprovider/works.go @@ -5,15 +5,6 @@ import ( "fmt" ) -type WorkProvider struct { - XMLProvider[Works] -} - -type Works struct { - XMLName xml.Name `xml:"werke"` - Work []Work `xml:"werk"` -} - type Work struct { XMLName xml.Name `xml:"werk"` URLs []URL `xml:"url"` @@ -32,24 +23,6 @@ type Citation struct { Inner } -func (w Works) Append(data Works) Works { - w.Work = append(w.Work, data.Work...) - return w -} - -func (w Works) String() string { - var res []string - for _, work := range w.Work { - res = append(res, work.String()) - } - - return fmt.Sprintf("Works: %v", res) -} - -func (w *Work) String() string { +func (w Work) String() string { return fmt.Sprintf("URLs: %v, Citation: %v, PreferredTitle: %s, Akteur: %v, Identifier: %v, AnnotationNote: %v\n", w.URLs, w.Citation, w.PreferredTitle, w.Akteur, w.Identifier, w.AnnotationNote) } - -func NewWorkProvider(paths []string) *WorkProvider { - return &WorkProvider{XMLProvider: XMLProvider[Works]{paths: paths}} -} diff --git a/providers/xmlprovider/xmlcommon.go b/providers/xmlprovider/xmlcommon.go index a52ebd6..834e9ab 100644 --- a/providers/xmlprovider/xmlcommon.go +++ b/providers/xmlprovider/xmlcommon.go @@ -39,6 +39,10 @@ type Identifier struct { ID string `xml:"id,attr"` } +func (i Identifier) GetIDs() []string { + return []string{i.ID} +} + type Reference struct { Ref string `xml:"ref,attr"` Category string `xml:"kat,attr"` diff --git a/providers/xmlprovider/xmlprovider.go b/providers/xmlprovider/xmlprovider.go index eab1b87..fc5fe81 100644 --- a/providers/xmlprovider/xmlprovider.go +++ b/providers/xmlprovider/xmlprovider.go @@ -10,34 +10,40 @@ import ( "github.com/Theodor-Springmann-Stiftung/kgpz_web/helpers/logging" ) -type KGPZXML[T any] interface { - Append(data T) T +type XMLItem interface { fmt.Stringer + GetIDs() []string } -type XMLProvider[T KGPZXML[T]] struct { - mu sync.Mutex - paths []string - Items T +type XMLProvider[T XMLItem] struct { + Paths []string + Items sync.Map + + mu sync.Mutex } type Library struct { - Agents *AgentProvider - Places *PlaceProvider - Works *WorkProvider - Categories *CategoryProvider - Issues *IssueProvider - Pieces *PieceProvider + Agents *XMLProvider[Agent] + Places *XMLProvider[Place] + Works *XMLProvider[Work] + Categories *XMLProvider[Category] + Issues *XMLProvider[Issue] + Pieces *XMLProvider[Piece] +} + +func (l *Library) String() string { + return fmt.Sprintf("Agents: %s\nPlaces: %s\nWorks: %s\nCategories: %s\nIssues: %s\nPieces: %s\n", + l.Agents.String(), l.Places.String(), l.Works.String(), l.Categories.String(), l.Issues.String(), l.Pieces.String()) } func NewLibrary(agentpaths, placepaths, workpaths, categorypaths, issuepaths, piecepaths []string) *Library { return &Library{ - Agents: NewAgentProvider(agentpaths), - Places: NewPlaceProvider(placepaths), - Works: NewWorkProvider(workpaths), - Categories: NewCategoryProvider(categorypaths), - Issues: NewIssueProvider(issuepaths), - Pieces: NewPieceProvider(piecepaths), + Agents: &XMLProvider[Agent]{Paths: agentpaths}, + Places: &XMLProvider[Place]{Paths: placepaths}, + Works: &XMLProvider[Work]{Paths: workpaths}, + Categories: &XMLProvider[Category]{Paths: categorypaths}, + Issues: &XMLProvider[Issue]{Paths: issuepaths}, + Pieces: &XMLProvider[Piece]{Paths: piecepaths}, } } @@ -47,84 +53,98 @@ func (l *Library) Serialize() { go func() { defer wg.Done() - err := l.Agents.Serialize() - if err != nil { - l.Agents = nil + lwg := sync.WaitGroup{} + for _, path := range l.Places.Paths { + lwg.Add(1) + go l.Places.Serialize(NewPlaceRoot(), path, &lwg) } + lwg.Wait() }() go func() { defer wg.Done() - err := l.Places.Serialize() - if err != nil { - l.Places = nil + lwg := sync.WaitGroup{} + for _, path := range l.Agents.Paths { + lwg.Add(1) + go l.Agents.Serialize(NewAgentRoot(), path, &lwg) } + lwg.Wait() }() go func() { defer wg.Done() - err := l.Works.Serialize() - if err != nil { - l.Works = nil + lwg := sync.WaitGroup{} + for _, path := range l.Categories.Paths { + lwg.Add(1) + go l.Categories.Serialize(NewCategoryRoot(), path, &lwg) } + lwg.Wait() }() go func() { defer wg.Done() - err := l.Categories.Serialize() - if err != nil { - l.Categories = nil + lwg := sync.WaitGroup{} + for _, path := range l.Works.Paths { + lwg.Add(1) + go l.Works.Serialize(NewWorkRoot(), path, &lwg) } + lwg.Wait() }() go func() { defer wg.Done() - err := l.Issues.Serialize() - if err != nil { - l.Issues = nil + lwg := sync.WaitGroup{} + for _, path := range l.Issues.Paths { + lwg.Add(1) + go l.Issues.Serialize(NewIssueRoot(), path, &lwg) } + lwg.Wait() }() go func() { defer wg.Done() - err := l.Pieces.Serialize() - if err != nil { - l.Pieces = nil + lwg := sync.WaitGroup{} + for _, path := range l.Pieces.Paths { + lwg.Add(1) + go l.Pieces.Serialize(NewPieceRoot(), path, &lwg) } + lwg.Wait() }() wg.Wait() } -// TODO: make Items into a sync.Map -func (p *XMLProvider[T]) Serialize() error { +func (p *XMLProvider[T]) Serialize(dataholder XMLRootElement[T], path string, wg *sync.WaitGroup) error { // Introduce goroutine for every path, locking on append: - var wg sync.WaitGroup - for _, path := range p.paths { - wg.Add(1) - go func(path string) { - defer wg.Done() - var data T - if err := UnmarshalFile(path, &data); err != nil { - return - } - p.mu.Lock() - defer p.mu.Unlock() - p.Items = p.Items.Append(data) - }(path) + if err := UnmarshalFile(path, dataholder); err != nil { + logging.Error(err, "Could not unmarshal file: "+path) + return err + } + for _, item := range dataholder.Children() { + // INFO: Mostly it's just one ID, so the double loop is not that bad. + for _, id := range item.GetIDs() { + p.Items.Store(id, item) + } + } + + if wg != nil { + wg.Done() } - wg.Wait() return nil } func (a *XMLProvider[T]) String() string { - a.mu.Lock() - defer a.mu.Unlock() - return fmt.Sprintf("Items: %s", a.Items) + var s string + a.Items.Range(func(key, value interface{}) bool { + v := value.(T) + s += v.String() + return true + }) + return s } -func UnmarshalFile[T any](filename string, data *T) error { +func UnmarshalFile[T any](filename string, data T) error { xmlFile, err := os.Open(filename) if err != nil { logging.Error(err, "Could not open file: "+filename) @@ -138,7 +158,7 @@ func UnmarshalFile[T any](filename string, data *T) error { logging.Error(err, "Could not read file: "+filename) return err } - err = xml.Unmarshal(byteValue, data) + err = xml.Unmarshal(byteValue, &data) if err != nil { logging.Error(err, "Could not unmarshal file: "+filename) @@ -146,3 +166,44 @@ func UnmarshalFile[T any](filename string, data *T) error { } return nil } + +func (p *XMLProvider[T]) Item(id string) *T { + item, ok := p.Items.Load(id) + if !ok { + return nil + } + + i := item.(T) + return &i +} + +func (p *XMLProvider[T]) Find(fn func(T) bool) []T { + var items []T + p.Items.Range(func(key, value interface{}) bool { + if fn(value.(T)) { + items = append(items, value.(T)) + } + return true + }) + return items +} + +func (p *XMLProvider[T]) FindKey(fn func(string) bool) []T { + var items []T + p.Items.Range(func(key, value interface{}) bool { + if fn(key.(string)) { + items = append(items, value.(T)) + } + return true + }) + return items +} + +func (p *XMLProvider[T]) All() []T { + var items []T + p.Items.Range(func(key, value interface{}) bool { + items = append(items, value.(T)) + return true + }) + return items +} diff --git a/server/server.go b/server/server.go index b7c66f7..cefc348 100644 --- a/server/server.go +++ b/server/server.go @@ -45,22 +45,23 @@ type Server struct { running chan bool shutdown *sync.WaitGroup cache *memory.Storage - - mu sync.Mutex - watcher *helpers.FileWatcher + engine *templating.Engine + mu sync.Mutex + watcher *helpers.FileWatcher kgpz *app.KGPZ } -func Create(k *app.KGPZ, c *providers.ConfigProvider) *Server { +func Create(k *app.KGPZ, c *providers.ConfigProvider, e *templating.Engine) *Server { if c == nil || k == nil { - logging.Error(nil, "Config or KGPZ is posssibly nil while tying to create server") + logging.Error(nil, "Error creating server: Config or App is posssibly nil.") return nil } return &Server{ Config: c, kgpz: k, + engine: e, } } @@ -96,12 +97,12 @@ func (s *Server) Watcher() error { } func (s *Server) Start() { + s.engine.Reload() + s.cache = memory.New(memory.Config{ GCInterval: 30 * time.Second, }) - engine := templating.NewEngine(&views.LayoutFS, &views.RoutesFS) - srv := fiber.New(fiber.Config{ AppName: s.Config.Address, CaseSensitive: false, @@ -118,7 +119,7 @@ func (s *Server) Start() { PassLocalsToViews: true, - Views: engine, + Views: s.engine, ViewsLayout: templating.DEFAULT_LAYOUT_NAME, }) diff --git a/templating/context.go b/templating/context.go index 9882359..348b167 100644 --- a/templating/context.go +++ b/templating/context.go @@ -92,13 +92,13 @@ func (c *TemplateContext) Globals() map[string]string { return c.globals } -func (c *TemplateContext) Template(fsys fs.FS) (*template.Template, error) { - t, err := readTemplates(fsys, nil, c.globals) +func (c *TemplateContext) Template(fsys fs.FS, funcmap *template.FuncMap) (*template.Template, error) { + t, err := readTemplates(fsys, nil, c.globals, funcmap) if err != nil { return nil, err } - t, err = readTemplates(fsys, t, c.locals) + t, err = readTemplates(fsys, t, c.locals, funcmap) if err != nil { return nil, err } @@ -106,14 +106,20 @@ func (c *TemplateContext) Template(fsys fs.FS) (*template.Template, error) { return t, nil } -func readTemplates(fsys fs.FS, t *template.Template, paths map[string]string) (*template.Template, error) { +func readTemplates(fsys fs.FS, t *template.Template, paths map[string]string, funcmap *template.FuncMap) (*template.Template, error) { for k, v := range paths { text, err := fs.ReadFile(fsys, v) if err != nil { return nil, NewError(FileAccessError, v) } - temp, err := template.New(k).Parse(string(text)) + temp := template.New(k) + + if funcmap != nil { + temp.Funcs(*funcmap) + } + + temp, err = temp.Parse(string(text)) if err != nil { return nil, err } diff --git a/templating/engine.go b/templating/engine.go index 871c651..4ed0836 100644 --- a/templating/engine.go +++ b/templating/engine.go @@ -5,6 +5,9 @@ import ( "io" "io/fs" "sync" + + "github.com/Theodor-Springmann-Stiftung/kgpz_web/app" + "github.com/Theodor-Springmann-Stiftung/kgpz_web/functions" ) type Engine struct { @@ -16,13 +19,28 @@ type Engine struct { FuncMap template.FuncMap } -func NewEngine(layouts, templates *fs.FS) *Engine { - return &Engine{ +// INFO: We pass the app here to be able to access the config and other data for functions +// which also means we must reload the engine if the app changes +func NewEngine(layouts, templates *fs.FS, app *app.KGPZ) *Engine { + e := Engine{ mu: &sync.Mutex{}, LayoutRegistry: NewLayoutRegistry(*layouts), TemplateRegistry: NewTemplateRegistry(*templates), - FuncMap: template.FuncMap{}, } + + e.MapFuncs(app) + return &e +} + +func (e *Engine) MapFuncs(app *app.KGPZ) error { + e.mu.Lock() + e.FuncMap = make(map[string]interface{}) + e.mu.Unlock() + + e.AddFunc("GetDate", functions.GetDate) + e.AddFunc("MonthName", functions.MonthName) + e.AddFunc("MonthNameShort", functions.MonthNameShort) + return nil } func (e *Engine) Load() error { @@ -43,18 +61,44 @@ func (e *Engine) Load() error { return nil } +func (e *Engine) Reload() error { + wg := sync.WaitGroup{} + wg.Add(2) + + go func() { + defer wg.Done() + e.LayoutRegistry.Reset() + }() + + go func() { + defer wg.Done() + e.TemplateRegistry.Reset() + }() + + wg.Wait() + return nil +} + +// INFO: fn is a function that returns either one value or two values, the second one being an error +func (e *Engine) AddFunc(name string, fn interface{}) { + e.mu.Lock() + defer e.mu.Unlock() + e.FuncMap[name] = fn +} + func (e *Engine) Render(out io.Writer, path string, data interface{}, layout ...string) error { // TODO: check if a reload is needed if files on disk have changed - + e.mu.Lock() + defer e.mu.Unlock() var l *template.Template if layout == nil || len(layout) == 0 { - lay, err := e.LayoutRegistry.Default() + lay, err := e.LayoutRegistry.Default(&e.FuncMap) if err != nil { return err } l = lay } else { - lay, err := e.LayoutRegistry.Layout(layout[0]) + lay, err := e.LayoutRegistry.Layout(layout[0], &e.FuncMap) if err != nil { return err } @@ -66,7 +110,7 @@ func (e *Engine) Render(out io.Writer, path string, data interface{}, layout ... return err } - err = e.TemplateRegistry.Add(path, lay) + err = e.TemplateRegistry.Add(path, lay, &e.FuncMap) if err != nil { return err } diff --git a/templating/layout_registry.go b/templating/layout_registry.go index 98add3e..c3410f2 100644 --- a/templating/layout_registry.go +++ b/templating/layout_registry.go @@ -33,6 +33,12 @@ func (r *LayoutRegistry) Register(fs fs.FS) *LayoutRegistry { return NewLayoutRegistry(merged_fs.MergeMultiple(fs, r.layoutsFS)) } +func (r *LayoutRegistry) Reset() error { + r.cache.Clear() + r.once = sync.Once{} + return nil +} + func (r *LayoutRegistry) Load() error { r.once.Do(func() { err := r.load() @@ -74,7 +80,7 @@ func (r *LayoutRegistry) load() error { return nil } -func (r *LayoutRegistry) Layout(name string) (*template.Template, error) { +func (r *LayoutRegistry) Layout(name string, funcmap *template.FuncMap) (*template.Template, error) { cached, ok := r.cache.Load(name) if ok { return cached.(*template.Template), nil @@ -87,7 +93,7 @@ func (r *LayoutRegistry) Layout(name string) (*template.Template, error) { return nil, NewError(NoTemplateError, name) } - t, err := context.Template(r.layoutsFS) + t, err := context.Template(r.layoutsFS, funcmap) if err != nil { return nil, err } @@ -97,6 +103,6 @@ func (r *LayoutRegistry) Layout(name string) (*template.Template, error) { return t, nil } -func (r *LayoutRegistry) Default() (*template.Template, error) { - return r.Layout(DEFAULT_LAYOUT_NAME) +func (r *LayoutRegistry) Default(funcmap *template.FuncMap) (*template.Template, error) { + return r.Layout(DEFAULT_LAYOUT_NAME, funcmap) } diff --git a/templating/template_registry.go b/templating/template_registry.go index 6d28287..1249a22 100644 --- a/templating/template_registry.go +++ b/templating/template_registry.go @@ -33,6 +33,12 @@ func (r *TemplateRegistry) Register(path string, fs fs.FS) *TemplateRegistry { return NewTemplateRegistry(merged_fs.MergeMultiple(fs, r.routesFS)) } +func (r *TemplateRegistry) Reset() error { + r.cache.Clear() + r.once = sync.Once{} + return nil +} + func (r *TemplateRegistry) Load() error { r.once.Do(func() { err := r.load() @@ -79,7 +85,7 @@ func (r *TemplateRegistry) load() error { // This function takes a template (typically a layout) and adds all the templates of // a given directory path to it. This is useful for adding a layout to a template. -func (r *TemplateRegistry) Add(path string, t *template.Template) error { +func (r *TemplateRegistry) Add(path string, t *template.Template, funcmap *template.FuncMap) error { temp, ok := r.cache.Load(path) if !ok { r.Load() @@ -88,14 +94,14 @@ func (r *TemplateRegistry) Add(path string, t *template.Template) error { return NewError(NoTemplateError, path) } - template, err := tc.Template(r.routesFS) + template, err := tc.Template(r.routesFS, funcmap) if err != nil { return err } r.cache.Store(path, template) - return r.Add(path, t) + return r.Add(path, t, funcmap) } casted := temp.(*template.Template) diff --git a/viewmodels/issue.go b/viewmodels/issue.go index 8aefc0d..2193bda 100644 --- a/viewmodels/issue.go +++ b/viewmodels/issue.go @@ -2,7 +2,6 @@ package viewmodels import ( "errors" - "strconv" "time" "github.com/Theodor-Springmann-Stiftung/kgpz_web/providers/xmlprovider" @@ -10,56 +9,22 @@ import ( const TLAYOUT = "2006-01-02" -var TRANSLM = map[string][]string{ - "January": {"Januar", "Jan", "1"}, - "February": {"Februar", "Feb", "2"}, - "March": {"März", "Mär", "3"}, - "April": {"April", "Apr", "4"}, - "May": {"Mai", "Mai", "5"}, - "June": {"Juni", "Jun", "6"}, - "July": {"Juli", "Jul", "7"}, - "August": {"August", "Aug", "8"}, - "September": {"September", "Sep", "9"}, - "October": {"Oktober", "Okt", "10"}, - "November": {"November", "Nov", "11"}, - "December": {"Dezember", "Dez", "12"}, -} - -var TRANSLD = map[string][]string{ - "Monday": {"Montag", "Mo"}, - "Tuesday": {"Dienstag", "Di"}, - "Wednesday": {"Mittwoch", "Mi"}, - "Thursday": {"Donnerstag", "Do"}, - "Friday": {"Freitag", "Fr"}, - "Saturday": {"Samstag", "Sa"}, - "Sunday": {"Sonntag", "So"}, -} - type IssueViewModel struct { xmlprovider.Issue - Weekday []string - Day int - Month []string + Day int + Month int + Year int } func IssueView(y string, No string, lib *xmlprovider.Library) (*IssueViewModel, error) { - n, err := strconv.Atoi(No) - if err != nil { - return nil, err + issue := lib.Issues.Item(y + "-" + No) + + if issue == nil { + return nil, errors.New("Issue not found") } - for _, i := range lib.Issues.Items.Issues { - if len(i.Datum.When) < 4 { - continue - } + return FromIssue(*issue) - d := i.Datum.When[:4] - if d == y && i.Number.No == n { - return FromIssue(i) - } - } - - return nil, errors.New("Issue not found") } func FromIssue(i xmlprovider.Issue) (*IssueViewModel, error) { @@ -69,10 +34,11 @@ func FromIssue(i xmlprovider.Issue) (*IssueViewModel, error) { } ivm := IssueViewModel{ - Issue: i, - Weekday: append(TRANSLD[t.Weekday().String()], t.Weekday().String()), - Day: t.Day(), - Month: TRANSLM[t.Month().String()]} + Issue: i, + Day: t.Day(), + Month: int(t.Month()), + Year: t.Year(), + } return &ivm, nil } diff --git a/viewmodels/singleissue.go b/viewmodels/singleissue.go index 01ad354..cf42d7a 100644 --- a/viewmodels/singleissue.go +++ b/viewmodels/singleissue.go @@ -1,6 +1,8 @@ package viewmodels import ( + "strconv" + "github.com/Theodor-Springmann-Stiftung/kgpz_web/helpers/logging" "github.com/Theodor-Springmann-Stiftung/kgpz_web/providers/xmlprovider" ) @@ -22,8 +24,10 @@ func NewSingleIssueView(y string, No string, lib *xmlprovider.Library) (*SingleI } sivm := SingleIssueViewModel{IssueViewModel: *ivm} + logging.Info(strconv.Itoa(len(lib.Pieces.All())) + "pieces in library") - for _, a := range lib.Pieces.Items.Piece { + lib.Pieces.Items.Range(func(key, value interface{}) bool { + a := value.(xmlprovider.Piece) for _, r := range a.IssueRefs { if r.Datum == y && r.Nr == No { p, err := NewPieceView(a) @@ -45,7 +49,8 @@ func NewSingleIssueView(y string, No string, lib *xmlprovider.Library) (*SingleI sivm.Additionals = append(sivm.Additionals, p) } } - } + return true + }) return &sivm, nil } diff --git a/viewmodels/year.go b/viewmodels/year.go index ca82955..81d4d88 100644 --- a/viewmodels/year.go +++ b/viewmodels/year.go @@ -21,9 +21,11 @@ func YearView(year string, lib *xmlprovider.Library) (*YearViewModel, error) { res := YearViewModel{Year: year} res.Issues = make(IssuesByMonth, 12) last := "" - for _, issue := range lib.Issues.Items.Issues { + + lib.Issues.Items.Range(func(key, value interface{}) bool { + issue := value.(xmlprovider.Issue) if len(issue.Datum.When) < 4 { - continue + return true } date := issue.Datum.When[0:4] @@ -35,7 +37,8 @@ func YearView(year string, lib *xmlprovider.Library) (*YearViewModel, error) { if date == year { res.PushIssue(issue) } - } + return true + }) if len(res.Issues) == 0 { return nil, errors.New("No issues found for year " + year) @@ -52,14 +55,12 @@ func (y *YearViewModel) PushIssue(i xmlprovider.Issue) { return } - month, _ := strconv.Atoi(iv.Month[2]) - - list, ok := y.Issues[month] + list, ok := y.Issues[iv.Month] if !ok { list = []IssueViewModel{} } - y.Issues[month] = append(list, *iv) + y.Issues[iv.Month] = append(list, *iv) } func (y *YearViewModel) PushAvailable(s string) { diff --git a/views/routes/body.tmpl b/views/routes/body.tmpl index 60ab96d..473f0cf 100644 --- a/views/routes/body.tmpl +++ b/views/routes/body.tmpl @@ -8,18 +8,21 @@ {{ $first := index $month 0 }} -

{{ index $first.Month 1 }}

+

{{ MonthNameShort $first.Month }}

{{ range $issue := $month }} + {{ $date := GetDate $issue.Datum.When }}
{{ $issue.Number.No }}
+
- {{ index $issue.Weekday 1 }} + {{ $date.Wd }}
-
{{ $issue.Day }}.{{ index $issue.Month 2 }}.
+ +
{{ $date.DayNo }}.{{ index $date.MonthNo }}.
 {{ end }} {{ end }}