Theodor-Springmann-Stiftung/kgpz_web
1
0
Fork 0
mirror of https://github.com/Theodor-Springmann-Stiftung/kgpz_web.git synced 2025-10-28 16:45:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

XML parsing overhaul

Browse Source
This commit is contained in:
Simon Martens
2024-11-22 00:35:27 +01:00
parent b93256c522
commit bc244fbad4
26 changed files with 507 additions and 352 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
app/kgpz.go
View File

@@ -85,8 +85,7 @@ func (k *KGPZ) Enrich() error {
}
// INFO: We pass agents by value since we don't want to block the library
agents := make([]xmlprovider.Agent, len(k.Library.Agents.Items.Agents))
_ = copy(agents, k.Library.Agents.Items.Agents)
agents := k.Library.Agents.All()
go func(agents []xmlprovider.Agent) {
k.GND.FetchPersons(agents)
k.GND.WriteCache(k.Config.GNDPath)

2
controllers/issue.go
View File

@@ -21,7 +21,7 @@ func GetIssue(kgpz *app.KGPZ) fiber.Handler {
return c.SendStatus(fiber.StatusNotFound)
}
issue, err := viewmodels.IssueView(y, d, kgpz.Library)
issue, err := viewmodels.NewSingleIssueView(y, d, kgpz.Library)
if err != nil {
logging.Error(err, "Issue could not be found")

66
functions/date.go Normal file
View File

@@ -0,0 +1,66 @@
package functions
import (
"time"
)
const TLAYOUT = "2006-01-02"
var TRANSLM = [][]string{
{"Januar", "Jan", "1"},
{"Februar", "Feb", "2"},
{"März", "Mär", "3"},
{"April", "Apr", "4"},
{"Mai", "Mai", "5"},
{"Juni", "Jun", "6"},
{"Juli", "Jul", "7"},
{"August", "Aug", "8"},
{"September", "Sep", "9"},
{"Oktober", "Okt", "10"},
{"November", "Nov", "11"},
{"Dezember", "Dez", "12"},
}
var TRANSLD = [][]string{
{"Montag", "Mo"},
{"Dienstag", "Di"},
{"Mittwoch", "Mi"},
{"Donnerstag", "Do"},
{"Freitag", "Fr"},
{"Samstag", "Sa"},
{"Sonntag", "So"},
}
type Date struct {
Month string
Mon string
MonthNo string
DayNo int
Weekday string
Wd string
}
func GetDate(d string) Date {
t, err := time.Parse(TLAYOUT, d)
if err != nil {
return Date{}
}
m := int(t.Month()) - 1
wd := int(t.Weekday()) - 1
return Date{
Month: TRANSLM[m][0],
Mon: TRANSLM[m][1],
MonthNo: TRANSLM[m][2],
DayNo: t.Day(),
Weekday: TRANSLD[wd][0],
Wd: TRANSLD[wd][1],
}
}
func MonthName(m int) string {
return TRANSLM[m-1][0]
}
func MonthNameShort(m int) string {
return TRANSLM[m-1][1]
}

2
go.mod
View File

@@ -22,7 +22,7 @@ require (
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-git/go-billy/v5 v5.5.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/google/uuid v1.5.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/kevinburke/ssh_config v1.2.0 // indirect
github.com/klauspost/compress v1.17.0 // indirect

2
go.sum
View File

@@ -46,6 +46,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=

6
kgpz_web.go
View File

@@ -11,6 +11,8 @@ import (
"github.com/Theodor-Springmann-Stiftung/kgpz_web/helpers/logging"
"github.com/Theodor-Springmann-Stiftung/kgpz_web/providers"
"github.com/Theodor-Springmann-Stiftung/kgpz_web/server"
"github.com/Theodor-Springmann-Stiftung/kgpz_web/templating"
"github.com/Theodor-Springmann-Stiftung/kgpz_web/views"
)
const (
@@ -44,7 +46,9 @@ func main() {
kgpz := app.NewKGPZ(cfg)
kgpz.Init()
server := server.Create(kgpz, cfg)
engine := templating.NewEngine(&views.LayoutFS, &views.RoutesFS, kgpz)
server := server.Create(kgpz, cfg, engine)
Start(kgpz, server)
}

9
providers/gnd/gnd.go
View File

@@ -25,6 +25,8 @@ type GNDProvider struct {
mu sync.Mutex
Persons sync.Map
// INFO: this holds all errors that occured during fetching
// and is used to prevent further fetches of the same person.
errmu sync.Mutex
errs map[string]int
}
@@ -98,7 +100,6 @@ func (p *GNDProvider) readPerson(file string) {
p.Persons.Store(person.Agent.GND, person)
return
}
}
func (p *GNDProvider) WriteCache(folder string) error {
@@ -134,6 +135,7 @@ func (p *GNDProvider) writePersons(folder string) error {
return nil
}
// INFO: this overwrites any existing files
func (p *GNDProvider) writePerson(folder, id string, person Person) {
// JSON marshalling of the person and sanity check:
filepath := filepath.Join(folder, person.KGPZID+".json")
@@ -209,9 +211,10 @@ func (p *GNDProvider) fetchPerson(person xmlprovider.Agent) {
var response *http.Response
// INFO: we do 3 retries with increasing time between them
for i := 0; i < 3; i++ {
response, err = http.DefaultClient.Do(request)
if err == nil && 400 > response.StatusCode {
if err == nil && response.StatusCode < 400 {
if i > 0 {
logging.Info("Successfully fetched person: " + person.ID + " after " + strconv.Itoa(i) + " retries")
}
@@ -245,7 +248,7 @@ func (p *GNDProvider) fetchPerson(person xmlprovider.Agent) {
return
}
// Wirte response body to file:
// For debug purposes: Write response body to file:
// os.WriteFile("gnd_responses/"+person.ID+".json", body, 0644)
gndPerson := Person{}

82
providers/gnd/model.go
View File

@@ -9,65 +9,65 @@ import (
type Person struct {
KGPZID string `json:"kgpzid"`
Agent xmlprovider.Agent `json:"agent"`
URL string `json:"id"`
DateOfBirth []string `json:"dateOfBirth"`
PlaceOfBirth []Entity `json:"placeOfBirth"`
DateOfDeath []string `json:"dateOfDeath"`
PlaceOfDeath []Entity `json:"placeOfDeath"`
PlaceOfBirthAsLiteral []string `json:"placeOfBirthAsLiteral"`
PlaceOfDeathAsLiteral []string `json:"placeOfDeathAsLiteral"`
BiographicalOrHistoricalInformation []string `json:"biographicalOrHistoricalInformation"`
PreferredName string `json:"preferredName"`
GndIdentifier string `json:"gndIdentifier"`
Wikipedia []Entity `json:"wikipedia"`
Depiction []Picture `json:"depiction"`
ProfessionOrOccupation []Entity `json:"professionOrOccupation"`
PreferredNameEntityForThePerson PersonNameEntity `json:"preferredNameEntityForThePerson"`
VariantNameEntityForThePerson []PersonNameEntity `json:"variantNameEntityForThePerson"`
VariantName []string `json:"variantName"`
SameAs []CrossReferences `json:"sameAs"`
Pseudonym []Entity `json:"pseudonym"`
GNDSubjectCategory []Entity `json:"gndSubjectCategory"`
Type []string `json:"type"`
PlaceOfActivity []Entity `json:"placeOfActivity"`
URL string `json:"id,omitempty"`
DateOfBirth []string `json:"dateOfBirth,omitempty"`
PlaceOfBirth []Entity `json:"placeOfBirth,omitempty"`
DateOfDeath []string `json:"dateOfDeath,omitempty"`
PlaceOfDeath []Entity `json:"placeOfDeath,omitempty"`
PlaceOfBirthAsLiteral []string `json:"placeOfBirthAsLiteral,omitempty"`
PlaceOfDeathAsLiteral []string `json:"placeOfDeathAsLiteral,omitempty"`
BiographicalOrHistoricalInformation []string `json:"biographicalOrHistoricalInformation,omitempty"`
PreferredName string `json:"preferredName,omitempty"`
GndIdentifier string `json:"gndIdentifier,omitempty"`
Wikipedia []Entity `json:"wikipedia,omitempty"`
Depiction []Picture `json:"depiction,omitempty"`
ProfessionOrOccupation []Entity `json:"professionOrOccupation,omitempty"`
PreferredNameEntityForThePerson PersonNameEntity `json:"preferredNameEntityForThePerson,omitempty"`
VariantNameEntityForThePerson []PersonNameEntity `json:"variantNameEntityForThePerson,omitempty"`
VariantName []string `json:"variantName,omitempty"`
SameAs []CrossReferences `json:"sameAs,omitempty"`
Pseudonym []Entity `json:"pseudonym,omitempty"`
GNDSubjectCategory []Entity `json:"gndSubjectCategory,omitempty"`
Type []string `json:"type,omitempty"`
PlaceOfActivity []Entity `json:"placeOfActivity,omitempty"`
}
type CrossReferences struct {
Items Collection `json:"collection"`
ID string `json:"id"`
Items Collection `json:"collection,omitempty"`
ID string `json:"id,omitempty"`
}
type Collection struct {
Abbr string `json:"abbr"`
Name string `json:"name"`
Publisher string `json:"publisher"`
Icon string `json:"icon"`
ID string `json:"id"`
Abbr string `json:"abbr,omitempty"`
Name string `json:"name,omitempty"`
Publisher string `json:"publisher,omitempty"`
Icon string `json:"icon,omitempty"`
ID string `json:"id,omitempty"`
}
type Link struct {
ID string `json:"id"`
Label string `json:"label"`
ID string `json:"id,omitempty"`
Label string `json:"label,omitempty"`
}
type Picture struct {
ID string `json:"id"`
URL string `json:"url"`
Thumbnail string `json:"thumbnail"`
ID string `json:"id,omitempty"`
URL string `json:"url,omitempty"`
Thumbnail string `json:"thumbnail,omitempty"`
}
type Entity struct {
ID string `json:"id"`
Label string `json:"label"`
ID string `json:"id,omitempty"`
Label string `json:"label,omitempty"`
}
type PersonNameEntity struct {
Prefix []string `json:"prefix"`
Counting []string `json:"counting"`
Forename []string `json:"forename"`
Surname []string `json:"surname"`
PersonalName []string `json:"personalName"`
NameAddition []string `json:"nameAddition"`
Prefix []string `json:"prefix,omitempty"`
Counting []string `json:"counting,omitempty"`
Forename []string `json:"forename,omitempty"`
Surname []string `json:"surname,omitempty"`
PersonalName []string `json:"personalName,omitempty"`
NameAddition []string `json:"nameAddition,omitempty"`
}
func (p Person) String() string {

29
providers/xmlprovider/agents.go
View File

@@ -5,10 +5,6 @@ import (
"fmt"
)
type AgentProvider struct {
XMLProvider[Agents]
}
type Agent struct {
XMLName xml.Name `xml:"akteur"`
Names []string `xml:"name"`
@@ -20,29 +16,6 @@ type Agent struct {
AnnotationNote
}
type Agents struct {
XMLName xml.Name `xml:"akteure"`
Agents []Agent `xml:"akteur"`
}
func (a Agents) String() string {
var res []string
for _, agent := range a.Agents {
res = append(res, agent.String())
}
return fmt.Sprintf("Agents: %v", res)
}
func (a Agents) Append(data Agents) Agents {
a.Agents = append(a.Agents, data.Agents...)
return a
}
func (a *Agent) String() string {
func (a Agent) String() string {
return fmt.Sprintf("ID: %s\nNames: %v\nSortName: %s\nLife: %s\nGND: %s\nAnnotations: %v\nNotes: %v\n", a.ID, a.Names, a.SortName, a.Life, a.GND, a.Annotations, a.Notes)
}
func NewAgentProvider(paths []string) *AgentProvider {
return &AgentProvider{XMLProvider: XMLProvider[Agents]{paths: paths}}
}

29
providers/xmlprovider/categories.go
View File

@@ -5,15 +5,6 @@ import (
"fmt"
)
type CategoryProvider struct {
XMLProvider[Categories]
}
type Categories struct {
XMLName xml.Name `xml:"kategorien"`
Category []Category `xml:"kategorie"`
}
type Category struct {
XMLName xml.Name `xml:"kategorie"`
Names []string `xml:"name"`
@@ -22,24 +13,6 @@ type Category struct {
AnnotationNote
}
func (c Categories) Append(data Categories) Categories {
c.Category = append(c.Category, data.Category...)
return c
}
func (c Categories) String() string {
var res []string
for _, category := range c.Category {
res = append(res, category.String())
}
return fmt.Sprintf("Categories: %v", res)
}
func (c *Category) String() string {
func (c Category) String() string {
return fmt.Sprintf("ID: %s\nNames: %v\nSortName: %s\nAnnotations: %v\nNotes: %v\n", c.ID, c.Names, c.SortName, c.Annotations, c.Notes)
}
func NewCategoryProvider(paths []string) *CategoryProvider {
return &CategoryProvider{XMLProvider: XMLProvider[Categories]{paths: paths}}
}

34
providers/xmlprovider/issues.go
View File

@@ -3,17 +3,9 @@ package xmlprovider
import (
"encoding/xml"
"fmt"
"strconv"
)
type IssueProvider struct {
XMLProvider[Issues]
}
type Issues struct {
XMLName xml.Name `xml:"stuecke"`
Issues []Issue `xml:"stueck"`
}
type Issue struct {
XMLName xml.Name `xml:"stueck"`
Number Nummer `xml:"nummer"`
@@ -37,24 +29,20 @@ type Additional struct {
Bis string `xml:"bis"`
}
func (i Issues) Append(data Issues) Issues {
i.Issues = append(i.Issues, data.Issues...)
return i
}
func (i Issues) String() string {
var res []string
for _, issue := range i.Issues {
res = append(res, issue.String())
func (i Issue) GetIDs() []string {
res := make([]string, 2)
date := i.Datum.When
if date != "" {
res = append(res, date)
}
return fmt.Sprintf("Issues: %v", res)
if len(date) > 4 {
res = append(res, i.Datum.When[0:4]+"-"+strconv.Itoa(i.Number.No))
}
return res
}
func (i Issue) String() string {
return fmt.Sprintf("Number: %v, Datum: %v, Von: %d, Bis: %d, Additionals: %v, Identifier: %v, AnnotationNote: %v\n", i.Number, i.Datum, i.Von, i.Bis, i.Additionals, i.Identifier, i.AnnotationNote)
}
func NewIssueProvider(paths []string) *IssueProvider {
return &IssueProvider{XMLProvider: XMLProvider[Issues]{paths: paths}}
}

37
providers/xmlprovider/pieces.go
View File

@@ -3,17 +3,10 @@ package xmlprovider
import (
"encoding/xml"
"fmt"
"github.com/google/uuid"
)
type PieceProvider struct {
XMLProvider[Pieces]
}
type Pieces struct {
XMLName xml.Name `xml:"beitraege"`
Piece []Piece `xml:"beitrag"`
}
type Piece struct {
XMLName xml.Name `xml:"beitrag"`
IssueRefs []IssueRef `xml:"stueck"`
@@ -30,24 +23,18 @@ type Piece struct {
AnnotationNote
}
func (p Pieces) Append(data Pieces) Pieces {
p.Piece = append(p.Piece, data.Piece...)
return p
}
func (p Pieces) String() string {
var res []string
for _, piece := range p.Piece {
res = append(res, piece.String())
}
return fmt.Sprintf("Pieces: %v", res)
}
func (p Piece) String() string {
return fmt.Sprintf("ID: %s\nIssueRefs: %v\nPlaceRefs: %v\nCategoryRefs: %v\nAgentRefs: %v\nWorkRefs: %v\nPieceRefs: %v\nAdditionalRef: %v\nIncipit: %v\nTitle: %v\nAnnotations: %v\nNotes: %v\n", p.ID, p.IssueRefs, p.PlaceRefs, p.CategoryRefs, p.AgentRefs, p.WorkRefs, p.PieceRefs, p.AdditionalRef, p.Incipit, p.Title, p.Annotations, p.Notes)
}
func NewPieceProvider(paths []string) *PieceProvider {
return &PieceProvider{XMLProvider: XMLProvider[Pieces]{paths: paths}}
func (p Piece) GetIDs() []string {
ret := make([]string, 2)
if p.ID != "" {
ret = append(ret, p.ID)
}
// TODO: sensible IDs
uid := uuid.New()
ret = append(ret, uid.String())
return ret
}

29
providers/xmlprovider/places.go
View File

@@ -5,15 +5,6 @@ import (
"fmt"
)
type PlaceProvider struct {
XMLProvider[Places]
}
type Places struct {
XMLName xml.Name `xml:"orte"`
Place []Place `xml:"ort"`
}
type Place struct {
XMLName xml.Name `xml:"ort"`
Names []string `xml:"name"`
@@ -23,24 +14,6 @@ type Place struct {
AnnotationNote
}
func (p Places) Append(data Places) Places {
p.Place = append(p.Place, data.Place...)
return p
}
func (p Places) String() string {
var res []string
for _, place := range p.Place {
res = append(res, place.String())
}
return fmt.Sprintf("Places: %v", res)
}
func (p *Place) String() string {
func (p Place) String() string {
return fmt.Sprintf("ID: %s\nNames: %v\nSortName: %s\nGeo: %s\nAnnotations: %v\nNotes: %v\n", p.ID, p.Names, p.SortName, p.Geo, p.Annotations, p.Notes)
}
func NewPlaceProvider(paths []string) *PlaceProvider {
return &PlaceProvider{XMLProvider: XMLProvider[Places]{paths: paths}}
}

111
providers/xmlprovider/roots.go Normal file
View File

@@ -0,0 +1,111 @@
package xmlprovider
import "encoding/xml"
// INFO: These are just root elements that hold the data of the XML files.
// They get discarded after a parse.
type XMLRootElement[T any] interface {
Children() []T
}
type AgentRoot struct {
XMLName xml.Name `xml:"akteure"`
Agents []Agent `xml:"akteur"`
}
func NewAgentRoot() *AgentRoot {
return &AgentRoot{}
}
func (a AgentRoot) New() *AgentRoot {
return NewAgentRoot()
}
func (a AgentRoot) Children() []Agent {
return a.Agents
}
type PlaceRoot struct {
XMLName xml.Name `xml:"orte"`
Place []Place `xml:"ort"`
}
func NewPlaceRoot() *PlaceRoot {
return &PlaceRoot{}
}
func (p PlaceRoot) New() *PlaceRoot {
return NewPlaceRoot()
}
func (p PlaceRoot) Children() []Place {
return p.Place
}
type CategoryRoot struct {
XMLName xml.Name `xml:"kategorien"`
Category []Category `xml:"kategorie"`
}
func NewCategoryRoot() *CategoryRoot {
return &CategoryRoot{}
}
func (c CategoryRoot) New() XMLRootElement[Category] {
return NewCategoryRoot()
}
func (c CategoryRoot) Children() []Category {
return c.Category
}
type PieceRoot struct {
XMLName xml.Name `xml:"beitraege"`
Piece []Piece `xml:"beitrag"`
}
func NewPieceRoot() *PieceRoot {
return &PieceRoot{}
}
func (p PieceRoot) New() XMLRootElement[Piece] {
return NewPieceRoot()
}
func (p PieceRoot) Children() []Piece {
return p.Piece
}
type IssueRoot struct {
XMLName xml.Name `xml:"stuecke"`
Issues []Issue `xml:"stueck"`
}
func NewIssueRoot() *IssueRoot {
return &IssueRoot{}
}
func (i IssueRoot) New() XMLRootElement[Issue] {
return NewIssueRoot()
}
func (i IssueRoot) Children() []Issue {
return i.Issues
}
type WorkRoot struct {
XMLName xml.Name `xml:"werke"`
Work []Work `xml:"werk"`
}
func NewWorkRoot() *WorkRoot {
return &WorkRoot{}
}
func (w WorkRoot) New() XMLRootElement[Work] {
return NewWorkRoot()
}
func (w WorkRoot) Children() []Work {
return w.Work
}

29
providers/xmlprovider/works.go
View File

@@ -5,15 +5,6 @@ import (
"fmt"
)
type WorkProvider struct {
XMLProvider[Works]
}
type Works struct {
XMLName xml.Name `xml:"werke"`
Work []Work `xml:"werk"`
}
type Work struct {
XMLName xml.Name `xml:"werk"`
URLs []URL `xml:"url"`
@@ -32,24 +23,6 @@ type Citation struct {
Inner
}
func (w Works) Append(data Works) Works {
w.Work = append(w.Work, data.Work...)
return w
}
func (w Works) String() string {
var res []string
for _, work := range w.Work {
res = append(res, work.String())
}
return fmt.Sprintf("Works: %v", res)
}
func (w *Work) String() string {
func (w Work) String() string {
return fmt.Sprintf("URLs: %v, Citation: %v, PreferredTitle: %s, Akteur: %v, Identifier: %v, AnnotationNote: %v\n", w.URLs, w.Citation, w.PreferredTitle, w.Akteur, w.Identifier, w.AnnotationNote)
}
func NewWorkProvider(paths []string) *WorkProvider {
return &WorkProvider{XMLProvider: XMLProvider[Works]{paths: paths}}
}

4
providers/xmlprovider/xmlcommon.go
View File

@@ -39,6 +39,10 @@ type Identifier struct {
ID string `xml:"id,attr"`
}
func (i Identifier) GetIDs() []string {
return []string{i.ID}
}
type Reference struct {
Ref string `xml:"ref,attr"`
Category string `xml:"kat,attr"`

171
providers/xmlprovider/xmlprovider.go
View File

@@ -10,34 +10,40 @@ import (
"github.com/Theodor-Springmann-Stiftung/kgpz_web/helpers/logging"
)
type KGPZXML[T any] interface {
Append(data T) T
type XMLItem interface {
fmt.Stringer
GetIDs() []string
}
type XMLProvider[T KGPZXML[T]] struct {
type XMLProvider[T XMLItem] struct {
Paths []string
Items sync.Map
mu sync.Mutex
paths []string
Items T
}
type Library struct {
Agents *AgentProvider
Places *PlaceProvider
Works *WorkProvider
Categories *CategoryProvider
Issues *IssueProvider
Pieces *PieceProvider
Agents *XMLProvider[Agent]
Places *XMLProvider[Place]
Works *XMLProvider[Work]
Categories *XMLProvider[Category]
Issues *XMLProvider[Issue]
Pieces *XMLProvider[Piece]
}
func (l *Library) String() string {
return fmt.Sprintf("Agents: %s\nPlaces: %s\nWorks: %s\nCategories: %s\nIssues: %s\nPieces: %s\n",
l.Agents.String(), l.Places.String(), l.Works.String(), l.Categories.String(), l.Issues.String(), l.Pieces.String())
}
func NewLibrary(agentpaths, placepaths, workpaths, categorypaths, issuepaths, piecepaths []string) *Library {
return &Library{
Agents: NewAgentProvider(agentpaths),
Places: NewPlaceProvider(placepaths),
Works: NewWorkProvider(workpaths),
Categories: NewCategoryProvider(categorypaths),
Issues: NewIssueProvider(issuepaths),
Pieces: NewPieceProvider(piecepaths),
Agents: &XMLProvider[Agent]{Paths: agentpaths},
Places: &XMLProvider[Place]{Paths: placepaths},
Works: &XMLProvider[Work]{Paths: workpaths},
Categories: &XMLProvider[Category]{Paths: categorypaths},
Issues: &XMLProvider[Issue]{Paths: issuepaths},
Pieces: &XMLProvider[Piece]{Paths: piecepaths},
}
}
@@ -47,84 +53,98 @@ func (l *Library) Serialize() {
go func() {
defer wg.Done()
err := l.Agents.Serialize()
if err != nil {
l.Agents = nil
lwg := sync.WaitGroup{}
for _, path := range l.Places.Paths {
lwg.Add(1)
go l.Places.Serialize(NewPlaceRoot(), path, &lwg)
}
lwg.Wait()
}()
go func() {
defer wg.Done()
err := l.Places.Serialize()
if err != nil {
l.Places = nil
lwg := sync.WaitGroup{}
for _, path := range l.Agents.Paths {
lwg.Add(1)
go l.Agents.Serialize(NewAgentRoot(), path, &lwg)
}
lwg.Wait()
}()
go func() {
defer wg.Done()
err := l.Works.Serialize()
if err != nil {
l.Works = nil
lwg := sync.WaitGroup{}
for _, path := range l.Categories.Paths {
lwg.Add(1)
go l.Categories.Serialize(NewCategoryRoot(), path, &lwg)
}
lwg.Wait()
}()
go func() {
defer wg.Done()
err := l.Categories.Serialize()
if err != nil {
l.Categories = nil
lwg := sync.WaitGroup{}
for _, path := range l.Works.Paths {
lwg.Add(1)
go l.Works.Serialize(NewWorkRoot(), path, &lwg)
}
lwg.Wait()
}()
go func() {
defer wg.Done()
err := l.Issues.Serialize()
if err != nil {
l.Issues = nil
lwg := sync.WaitGroup{}
for _, path := range l.Issues.Paths {
lwg.Add(1)
go l.Issues.Serialize(NewIssueRoot(), path, &lwg)
}
lwg.Wait()
}()
go func() {
defer wg.Done()
err := l.Pieces.Serialize()
if err != nil {
l.Pieces = nil
lwg := sync.WaitGroup{}
for _, path := range l.Pieces.Paths {
lwg.Add(1)
go l.Pieces.Serialize(NewPieceRoot(), path, &lwg)
}
lwg.Wait()
}()
wg.Wait()
}
// TODO: make Items into a sync.Map
func (p *XMLProvider[T]) Serialize() error {
func (p *XMLProvider[T]) Serialize(dataholder XMLRootElement[T], path string, wg *sync.WaitGroup) error {
// Introduce goroutine for every path, locking on append:
var wg sync.WaitGroup
for _, path := range p.paths {
wg.Add(1)
go func(path string) {
defer wg.Done()
var data T
if err := UnmarshalFile(path, &data); err != nil {
return
if err := UnmarshalFile(path, dataholder); err != nil {
logging.Error(err, "Could not unmarshal file: "+path)
return err
}
p.mu.Lock()
defer p.mu.Unlock()
p.Items = p.Items.Append(data)
}(path)
for _, item := range dataholder.Children() {
// INFO: Mostly it's just one ID, so the double loop is not that bad.
for _, id := range item.GetIDs() {
p.Items.Store(id, item)
}
}
if wg != nil {
wg.Done()
}
wg.Wait()
return nil
}
func (a *XMLProvider[T]) String() string {
a.mu.Lock()
defer a.mu.Unlock()
return fmt.Sprintf("Items: %s", a.Items)
var s string
a.Items.Range(func(key, value interface{}) bool {
v := value.(T)
s += v.String()
return true
})
return s
}
func UnmarshalFile[T any](filename string, data *T) error {
func UnmarshalFile[T any](filename string, data T) error {
xmlFile, err := os.Open(filename)
if err != nil {
logging.Error(err, "Could not open file: "+filename)
@@ -138,7 +158,7 @@ func UnmarshalFile[T any](filename string, data *T) error {
logging.Error(err, "Could not read file: "+filename)
return err
}
err = xml.Unmarshal(byteValue, data)
err = xml.Unmarshal(byteValue, &data)
if err != nil {
logging.Error(err, "Could not unmarshal file: "+filename)
@@ -146,3 +166,44 @@ func UnmarshalFile[T any](filename string, data *T) error {
}
return nil
}
func (p *XMLProvider[T]) Item(id string) *T {
item, ok := p.Items.Load(id)
if !ok {
return nil
}
i := item.(T)
return &i
}
func (p *XMLProvider[T]) Find(fn func(T) bool) []T {
var items []T
p.Items.Range(func(key, value interface{}) bool {
if fn(value.(T)) {
items = append(items, value.(T))
}
return true
})
return items
}
func (p *XMLProvider[T]) FindKey(fn func(string) bool) []T {
var items []T
p.Items.Range(func(key, value interface{}) bool {
if fn(key.(string)) {
items = append(items, value.(T))
}
return true
})
return items
}
func (p *XMLProvider[T]) All() []T {
var items []T
p.Items.Range(func(key, value interface{}) bool {
items = append(items, value.(T))
return true
})
return items
}

13
server/server.go
View File

@@ -45,22 +45,23 @@ type Server struct {
running chan bool
shutdown *sync.WaitGroup
cache *memory.Storage
engine *templating.Engine
mu sync.Mutex
watcher *helpers.FileWatcher
kgpz *app.KGPZ
}
func Create(k *app.KGPZ, c *providers.ConfigProvider) *Server {
func Create(k *app.KGPZ, c *providers.ConfigProvider, e *templating.Engine) *Server {
if c == nil || k == nil {
logging.Error(nil, "Config or KGPZ is posssibly nil while tying to create server")
logging.Error(nil, "Error creating server: Config or App is posssibly nil.")
return nil
}
return &Server{
Config: c,
kgpz: k,
engine: e,
}
}
@@ -96,12 +97,12 @@ func (s *Server) Watcher() error {
}
func (s *Server) Start() {
s.engine.Reload()
s.cache = memory.New(memory.Config{
GCInterval: 30 * time.Second,
})
engine := templating.NewEngine(&views.LayoutFS, &views.RoutesFS)
srv := fiber.New(fiber.Config{
AppName: s.Config.Address,
CaseSensitive: false,
@@ -118,7 +119,7 @@ func (s *Server) Start() {
PassLocalsToViews: true,
Views: engine,
Views: s.engine,
ViewsLayout: templating.DEFAULT_LAYOUT_NAME,
})

16
templating/context.go
View File

@@ -92,13 +92,13 @@ func (c *TemplateContext) Globals() map[string]string {
return c.globals
}
func (c *TemplateContext) Template(fsys fs.FS) (*template.Template, error) {
t, err := readTemplates(fsys, nil, c.globals)
func (c *TemplateContext) Template(fsys fs.FS, funcmap *template.FuncMap) (*template.Template, error) {
t, err := readTemplates(fsys, nil, c.globals, funcmap)
if err != nil {
return nil, err
}
t, err = readTemplates(fsys, t, c.locals)
t, err = readTemplates(fsys, t, c.locals, funcmap)
if err != nil {
return nil, err
}
@@ -106,14 +106,20 @@ func (c *TemplateContext) Template(fsys fs.FS) (*template.Template, error) {
return t, nil
}
func readTemplates(fsys fs.FS, t *template.Template, paths map[string]string) (*template.Template, error) {
func readTemplates(fsys fs.FS, t *template.Template, paths map[string]string, funcmap *template.FuncMap) (*template.Template, error) {
for k, v := range paths {
text, err := fs.ReadFile(fsys, v)
if err != nil {
return nil, NewError(FileAccessError, v)
}
temp, err := template.New(k).Parse(string(text))
temp := template.New(k)
if funcmap != nil {
temp.Funcs(*funcmap)
}
temp, err = temp.Parse(string(text))
if err != nil {
return nil, err
}

58
templating/engine.go
View File

@@ -5,6 +5,9 @@ import (
"io"
"io/fs"
"sync"
"github.com/Theodor-Springmann-Stiftung/kgpz_web/app"
"github.com/Theodor-Springmann-Stiftung/kgpz_web/functions"
)
type Engine struct {
@@ -16,13 +19,28 @@ type Engine struct {
FuncMap template.FuncMap
}
func NewEngine(layouts, templates *fs.FS) *Engine {
return &Engine{
// INFO: We pass the app here to be able to access the config and other data for functions
// which also means we must reload the engine if the app changes
func NewEngine(layouts, templates *fs.FS, app *app.KGPZ) *Engine {
e := Engine{
mu: &sync.Mutex{},
LayoutRegistry: NewLayoutRegistry(*layouts),
TemplateRegistry: NewTemplateRegistry(*templates),
FuncMap: template.FuncMap{},
}
e.MapFuncs(app)
return &e
}
func (e *Engine) MapFuncs(app *app.KGPZ) error {
e.mu.Lock()
e.FuncMap = make(map[string]interface{})
e.mu.Unlock()
e.AddFunc("GetDate", functions.GetDate)
e.AddFunc("MonthName", functions.MonthName)
e.AddFunc("MonthNameShort", functions.MonthNameShort)
return nil
}
func (e *Engine) Load() error {
@@ -43,18 +61,44 @@ func (e *Engine) Load() error {
return nil
}
func (e *Engine) Reload() error {
wg := sync.WaitGroup{}
wg.Add(2)
go func() {
defer wg.Done()
e.LayoutRegistry.Reset()
}()
go func() {
defer wg.Done()
e.TemplateRegistry.Reset()
}()
wg.Wait()
return nil
}
// INFO: fn is a function that returns either one value or two values, the second one being an error
func (e *Engine) AddFunc(name string, fn interface{}) {
e.mu.Lock()
defer e.mu.Unlock()
e.FuncMap[name] = fn
}
func (e *Engine) Render(out io.Writer, path string, data interface{}, layout ...string) error {
// TODO: check if a reload is needed if files on disk have changed
e.mu.Lock()
defer e.mu.Unlock()
var l *template.Template
if layout == nil || len(layout) == 0 {
lay, err := e.LayoutRegistry.Default()
lay, err := e.LayoutRegistry.Default(&e.FuncMap)
if err != nil {
return err
}
l = lay
} else {
lay, err := e.LayoutRegistry.Layout(layout[0])
lay, err := e.LayoutRegistry.Layout(layout[0], &e.FuncMap)
if err != nil {
return err
}
@@ -66,7 +110,7 @@ func (e *Engine) Render(out io.Writer, path string, data interface{}, layout ...
return err
}
err = e.TemplateRegistry.Add(path, lay)
err = e.TemplateRegistry.Add(path, lay, &e.FuncMap)
if err != nil {
return err
}

14
templating/layout_registry.go
View File

@@ -33,6 +33,12 @@ func (r *LayoutRegistry) Register(fs fs.FS) *LayoutRegistry {
return NewLayoutRegistry(merged_fs.MergeMultiple(fs, r.layoutsFS))
}
func (r *LayoutRegistry) Reset() error {
r.cache.Clear()
r.once = sync.Once{}
return nil
}
func (r *LayoutRegistry) Load() error {
r.once.Do(func() {
err := r.load()
@@ -74,7 +80,7 @@ func (r *LayoutRegistry) load() error {
return nil
}
func (r *LayoutRegistry) Layout(name string) (*template.Template, error) {
func (r *LayoutRegistry) Layout(name string, funcmap *template.FuncMap) (*template.Template, error) {
cached, ok := r.cache.Load(name)
if ok {
return cached.(*template.Template), nil
@@ -87,7 +93,7 @@ func (r *LayoutRegistry) Layout(name string) (*template.Template, error) {
return nil, NewError(NoTemplateError, name)
}
t, err := context.Template(r.layoutsFS)
t, err := context.Template(r.layoutsFS, funcmap)
if err != nil {
return nil, err
}
@@ -97,6 +103,6 @@ func (r *LayoutRegistry) Layout(name string) (*template.Template, error) {
return t, nil
}
func (r *LayoutRegistry) Default() (*template.Template, error) {
return r.Layout(DEFAULT_LAYOUT_NAME)
func (r *LayoutRegistry) Default(funcmap *template.FuncMap) (*template.Template, error) {
return r.Layout(DEFAULT_LAYOUT_NAME, funcmap)
}

12
templating/template_registry.go
View File

@@ -33,6 +33,12 @@ func (r *TemplateRegistry) Register(path string, fs fs.FS) *TemplateRegistry {
return NewTemplateRegistry(merged_fs.MergeMultiple(fs, r.routesFS))
}
func (r *TemplateRegistry) Reset() error {
r.cache.Clear()
r.once = sync.Once{}
return nil
}
func (r *TemplateRegistry) Load() error {
r.once.Do(func() {
err := r.load()
@@ -79,7 +85,7 @@ func (r *TemplateRegistry) load() error {
// This function takes a template (typically a layout) and adds all the templates of
// a given directory path to it. This is useful for adding a layout to a template.
func (r *TemplateRegistry) Add(path string, t *template.Template) error {
func (r *TemplateRegistry) Add(path string, t *template.Template, funcmap *template.FuncMap) error {
temp, ok := r.cache.Load(path)
if !ok {
r.Load()
@@ -88,14 +94,14 @@ func (r *TemplateRegistry) Add(path string, t *template.Template) error {
return NewError(NoTemplateError, path)
}
template, err := tc.Template(r.routesFS)
template, err := tc.Template(r.routesFS, funcmap)
if err != nil {
return err
}
r.cache.Store(path, template)
return r.Add(path, t)
return r.Add(path, t, funcmap)
}
casted := temp.(*template.Template)

56
viewmodels/issue.go
View File

@@ -2,7 +2,6 @@ package viewmodels
import (
"errors"
"strconv"
"time"
"github.com/Theodor-Springmann-Stiftung/kgpz_web/providers/xmlprovider"
@@ -10,56 +9,22 @@ import (
const TLAYOUT = "2006-01-02"
var TRANSLM = map[string][]string{
"January": {"Januar", "Jan", "1"},
"February": {"Februar", "Feb", "2"},
"March": {"März", "Mär", "3"},
"April": {"April", "Apr", "4"},
"May": {"Mai", "Mai", "5"},
"June": {"Juni", "Jun", "6"},
"July": {"Juli", "Jul", "7"},
"August": {"August", "Aug", "8"},
"September": {"September", "Sep", "9"},
"October": {"Oktober", "Okt", "10"},
"November": {"November", "Nov", "11"},
"December": {"Dezember", "Dez", "12"},
}
var TRANSLD = map[string][]string{
"Monday": {"Montag", "Mo"},
"Tuesday": {"Dienstag", "Di"},
"Wednesday": {"Mittwoch", "Mi"},
"Thursday": {"Donnerstag", "Do"},
"Friday": {"Freitag", "Fr"},
"Saturday": {"Samstag", "Sa"},
"Sunday": {"Sonntag", "So"},
}
type IssueViewModel struct {
xmlprovider.Issue
Weekday []string
Day int
Month []string
Month int
Year int
}
func IssueView(y string, No string, lib *xmlprovider.Library) (*IssueViewModel, error) {
n, err := strconv.Atoi(No)
if err != nil {
return nil, err
}
for _, i := range lib.Issues.Items.Issues {
if len(i.Datum.When) < 4 {
continue
}
d := i.Datum.When[:4]
if d == y && i.Number.No == n {
return FromIssue(i)
}
}
issue := lib.Issues.Item(y + "-" + No)
if issue == nil {
return nil, errors.New("Issue not found")
}
return FromIssue(*issue)
}
func FromIssue(i xmlprovider.Issue) (*IssueViewModel, error) {
@@ -70,9 +35,10 @@ func FromIssue(i xmlprovider.Issue) (*IssueViewModel, error) {
ivm := IssueViewModel{
Issue: i,
Weekday: append(TRANSLD[t.Weekday().String()], t.Weekday().String()),
Day: t.Day(),
Month: TRANSLM[t.Month().String()]}
Month: int(t.Month()),
Year: t.Year(),
}
return &ivm, nil
}

9
viewmodels/singleissue.go
View File

@@ -1,6 +1,8 @@
package viewmodels
import (
"strconv"
"github.com/Theodor-Springmann-Stiftung/kgpz_web/helpers/logging"
"github.com/Theodor-Springmann-Stiftung/kgpz_web/providers/xmlprovider"
)
@@ -22,8 +24,10 @@ func NewSingleIssueView(y string, No string, lib *xmlprovider.Library) (*SingleI
}
sivm := SingleIssueViewModel{IssueViewModel: *ivm}
logging.Info(strconv.Itoa(len(lib.Pieces.All())) + "pieces in library")
for _, a := range lib.Pieces.Items.Piece {
lib.Pieces.Items.Range(func(key, value interface{}) bool {
a := value.(xmlprovider.Piece)
for _, r := range a.IssueRefs {
if r.Datum == y && r.Nr == No {
p, err := NewPieceView(a)
@@ -45,7 +49,8 @@ func NewSingleIssueView(y string, No string, lib *xmlprovider.Library) (*SingleI
sivm.Additionals = append(sivm.Additionals, p)
}
}
}
return true
})
return &sivm, nil
}

15
viewmodels/year.go
View File

@@ -21,9 +21,11 @@ func YearView(year string, lib *xmlprovider.Library) (*YearViewModel, error) {
res := YearViewModel{Year: year}
res.Issues = make(IssuesByMonth, 12)
last := ""
for _, issue := range lib.Issues.Items.Issues {
lib.Issues.Items.Range(func(key, value interface{}) bool {
issue := value.(xmlprovider.Issue)
if len(issue.Datum.When) < 4 {
continue
return true
}
date := issue.Datum.When[0:4]
@@ -35,7 +37,8 @@ func YearView(year string, lib *xmlprovider.Library) (*YearViewModel, error) {
if date == year {
res.PushIssue(issue)
}
}
return true
})
if len(res.Issues) == 0 {
return nil, errors.New("No issues found for year " + year)
@@ -52,14 +55,12 @@ func (y *YearViewModel) PushIssue(i xmlprovider.Issue) {
return
}
month, _ := strconv.Atoi(iv.Month[2])
list, ok := y.Issues[month]
list, ok := y.Issues[iv.Month]
if !ok {
list = []IssueViewModel{}
}
y.Issues[month] = append(list, *iv)
y.Issues[iv.Month] = append(list, *iv)
}
func (y *YearViewModel) PushAvailable(s string) {

9
views/routes/body.tmpl
View File

@@ -8,18 +8,21 @@
<!-- Month Header -->
{{ $first := index $month 0 }}
<h2>{{ index $first.Month 1 }}</h2>
<h2>{{ MonthNameShort $first.Month }}</h2>
<!-- Issues -->
{{ range $issue := $month }}
{{ $date := GetDate $issue.Datum.When }}
<a href="/{{ $y }}/{{ $issue.Number.No }}">
<div>
{{ $issue.Number.No }}
</div>
<div>
{{ index $issue.Weekday 1 }}
{{ $date.Wd }}
</div>
<div>{{ $issue.Day }}.{{ index $issue.Month 2 }}.</div>
<div>{{ $date.DayNo }}.{{ index $date.MonthNo }}.</div>
</a>
{{ end }}
{{ end }}