diff --git a/app/kgpz.go b/app/kgpz.go index 9ee537d..6672e0f 100644 --- a/app/kgpz.go +++ b/app/kgpz.go @@ -27,10 +27,11 @@ const ( PRIVACY_URL = "/datenschutz/" CONTACT_URL = "/kontakt/" CITATION_URL = "/zitation/" + SEARCH_URL = "/suche/" - INDEX_URL = "/1764" + INDEX_URL = "/jahrgang/1764" - YEAR_OVERVIEW_URL = "/:year" + YEAR_OVERVIEW_URL = "/jahrgang/:year" PLACE_OVERVIEW_URL = "/ort/:place" AGENTS_OVERVIEW_URL = "/akteure/:letterorid" CATEGORY_OVERVIEW_URL = "/kategorie/:category" @@ -109,7 +110,13 @@ func (k *KGPZ) Init() error { k.Enrich() go k.Pull() - k.BuildSearchIndex() + err := k.Search.LoadIndeces() + if err != nil { + logging.Error(err, "Error loading search indeces.") + k.BuildSearchIndex() + } else { + logging.Info("Search indeces loaded.") + } return nil } @@ -125,6 +132,7 @@ func (k *KGPZ) Routes(srv *fiber.App) error { return nil }) + srv.Get(SEARCH_URL, controllers.GetSearch(k.Library, k.Search)) srv.Get(PLACE_OVERVIEW_URL, controllers.GetPlace(k.Library)) srv.Get(CATEGORY_OVERVIEW_URL, controllers.GetCategory(k.Library)) srv.Get(AGENTS_OVERVIEW_URL, controllers.GetAgents(k.Library)) diff --git a/controllers/search.go b/controllers/search.go new file mode 100644 index 0000000..e95cc93 --- /dev/null +++ b/controllers/search.go @@ -0,0 +1,26 @@ +package controllers + +import ( + "github.com/Theodor-Springmann-Stiftung/kgpz_web/helpers/logging" + searchprovider "github.com/Theodor-Springmann-Stiftung/kgpz_web/providers/search" + "github.com/Theodor-Springmann-Stiftung/kgpz_web/viewmodels" + "github.com/Theodor-Springmann-Stiftung/kgpz_web/xmlmodels" + "github.com/gofiber/fiber/v2" +) + +func GetSearch(kgpz *xmlmodels.Library, sp *searchprovider.SearchProvider) fiber.Handler { + return func(c *fiber.Ctx) error { + search := c.Query("q") + if search == "" { + return c.SendStatus(fiber.StatusNotFound) + } + + view, err := viewmodels.NewSearchView(search, kgpz, sp) + if err != nil { + logging.Error(err) + return c.SendStatus(fiber.StatusNotFound) + } + + return c.Render("/search/", fiber.Map{"model": view, "search": search}) + } +} diff --git a/helpers/datatypes/string.go b/helpers/datatypes/string.go new file mode 100644 index 0000000..52da60b --- /dev/null +++ b/helpers/datatypes/string.go @@ -0,0 +1,45 @@ +package datatypes + +import ( + "regexp" + "strings" + "unicode" +) + +var html_regexp = regexp.MustCompile(`<[^>]+>`) +var ws_regexp = regexp.MustCompile(`\s+`) + +func DeleteTags(s string) string { + return html_regexp.ReplaceAllString(s, "") +} + +func NormalizeString(s string) string { + s = strings.TrimSpace(s) + s = strings.ReplaceAll(s, "
", "") + s = strings.ReplaceAll(s, "
", "") + return s +} + +func SliceJoin[T any](slice []T, join string, f func(T) string) string { + var result []string + for _, item := range slice { + ap := f(item) + if ap != "" { + result = append(result, ap) + } + } + return strings.Join(result, join) +} + +func RemovePunctuation(s string) string { + return strings.Map(func(r rune) rune { + if unicode.IsPunct(r) { + return -1 + } + return r + }, s) +} + +func NormalizeWhitespace(s string) string { + return strings.TrimSpace(ws_regexp.ReplaceAllString(s, " ")) +} diff --git a/providers/search/searchprovider.go b/providers/search/searchprovider.go index 61e21c8..7b9c724 100644 --- a/providers/search/searchprovider.go +++ b/providers/search/searchprovider.go @@ -50,7 +50,31 @@ func (sp *SearchProvider) Index(item ISearchable, lib *xmlmodels.Library) error return err } - return i.Index(keys[0], item.Readable(lib)) + read := item.Readable(lib) + return i.Index(keys[0], read) +} + +// TODO: this is sloppy +func (sp *SearchProvider) LoadIndeces() error { + files, err := filepath.Glob(filepath.Join(sp.basepath, "*.bleve")) + if err != nil { + return err + } + + if len(files) == 0 { + return errors.New("No indeces found.") + } + + for _, file := range files { + index, err := bleve.Open(file) + if err != nil { + return err + } + typ := filepath.Base(file) + typ = typ[:len(typ)-6] + sp.indeces.Store(typ, index) + } + return nil } func (sp *SearchProvider) FindCreateIndex(typ string) (bleve.Index, error) { @@ -77,6 +101,16 @@ func (sp *SearchProvider) FindCreateIndex(typ string) (bleve.Index, error) { return ind, nil } +func (sp *SearchProvider) GetIndex(typ string) (bleve.Index, error) { + index, ok := sp.indeces.Load(typ) + if !ok { + return nil, errors.New("Index not found.") + } + + i := index.(bleve.Index) + return i, nil +} + func default_mapping() (*mapping.IndexMappingImpl, error) { indexMapping := bleve.NewIndexMapping() diff --git a/providers/xmlprovider/models.go b/providers/xmlprovider/models.go index 0afb9f9..566741b 100644 --- a/providers/xmlprovider/models.go +++ b/providers/xmlprovider/models.go @@ -8,7 +8,7 @@ type IXMLItem interface { // - Keys should be unique // - Keys[0] has the special meaning of the primary key (for FTS etc.) Keys() []string - Name() string + Type() string } type ILibrary interface { diff --git a/providers/xmlprovider/xmlprovider.go b/providers/xmlprovider/xmlprovider.go index b4c4a67..140a5fc 100644 --- a/providers/xmlprovider/xmlprovider.go +++ b/providers/xmlprovider/xmlprovider.go @@ -147,7 +147,7 @@ func (p *XMLProvider[T]) ReverseLookup(item IXMLItem) []Resolved[T] { keys := item.Keys() for _, key := range keys { - r, err := p.Resolver.Get(item.Name(), key) + r, err := p.Resolver.Get(item.Type(), key) if err == nil { ret = append(ret, r...) } diff --git a/viewmodels/searchview.go b/viewmodels/searchview.go new file mode 100644 index 0000000..195c629 --- /dev/null +++ b/viewmodels/searchview.go @@ -0,0 +1,153 @@ +package viewmodels + +import ( + "fmt" + "sync" + + "github.com/Theodor-Springmann-Stiftung/kgpz_web/helpers/datatypes" + searchprovider "github.com/Theodor-Springmann-Stiftung/kgpz_web/providers/search" + "github.com/Theodor-Springmann-Stiftung/kgpz_web/xmlmodels" + "github.com/blevesearch/bleve/v2" + "golang.org/x/text/cases" + "golang.org/x/text/language" + "golang.org/x/text/unicode/norm" +) + +type Result[T any] struct { + Count uint64 + Items []T +} + +type SearchView struct { + Agents Result[xmlmodels.Agent] + Works Result[xmlmodels.Work] + Places Result[xmlmodels.Place] + Categories Result[xmlmodels.Category] + Pieces Result[xmlmodels.Piece] + Issues Result[xmlmodels.Issue] +} + +func NewSearchView(search string, kgpz *xmlmodels.Library, sp *searchprovider.SearchProvider) (*SearchView, error) { + sw := SearchView{} + search = datatypes.DeleteTags(search) + search = datatypes.NormalizeString(search) + search = datatypes.RemovePunctuation(search) + search = cases.Lower(language.German).String(search) + search = norm.NFKD.String(search) + + query := bleve.NewTermQuery(search) + request := bleve.NewSearchRequest(query) + request.Size = 100 + + agentIndex, erragent := sp.GetIndex(xmlmodels.AGENT_TYPE) + workIndex, errwork := sp.GetIndex(xmlmodels.WORK_TYPE) + placeIndex, errplace := sp.GetIndex(xmlmodels.PLACE_TYPE) + categoryIndex, errcategory := sp.GetIndex(xmlmodels.CATEGORY_TYPE) + pieceIndex, errpiece := sp.GetIndex(xmlmodels.PIECE_TYPE) + issueIndex, errissue := sp.GetIndex(xmlmodels.ISSUE_TYPE) + + if agentIndex == nil || workIndex == nil || placeIndex == nil || categoryIndex == nil || pieceIndex == nil || issueIndex == nil { + return nil, fmt.Errorf("Indeces not found.") + } + + wg := sync.WaitGroup{} + if erragent == nil { + wg.Add(1) + go func() { + agentResults, _ := agentIndex.Search(request) + result := Result[xmlmodels.Agent]{Count: agentResults.Total} + for _, hit := range agentResults.Hits { + agent := kgpz.Agents.Item(hit.ID) + if agent != nil { + result.Items = append(result.Items, *agent) + } + } + sw.Agents = result + wg.Done() + }() + } + + if errwork == nil { + wg.Add(1) + go func() { + workResults, _ := workIndex.Search(request) + result := Result[xmlmodels.Work]{Count: workResults.Total} + for _, hit := range workResults.Hits { + work := kgpz.Works.Item(hit.ID) + if work != nil { + result.Items = append(result.Items, *work) + } + } + sw.Works = result + wg.Done() + }() + } + + if errplace == nil { + wg.Add(1) + go func() { + placeResults, _ := placeIndex.Search(request) + result := Result[xmlmodels.Place]{Count: placeResults.Total} + for _, hit := range placeResults.Hits { + place := kgpz.Places.Item(hit.ID) + if place != nil { + result.Items = append(result.Items, *place) + } + } + sw.Places = result + wg.Done() + }() + } + + if errcategory == nil { + wg.Add(1) + go func() { + categoryResults, _ := categoryIndex.Search(request) + result := Result[xmlmodels.Category]{Count: categoryResults.Total} + for _, hit := range categoryResults.Hits { + category := kgpz.Categories.Item(hit.ID) + if category != nil { + result.Items = append(result.Items, *category) + } + } + sw.Categories = result + wg.Done() + }() + } + + if errpiece == nil { + wg.Add(1) + go func() { + pieceResults, _ := pieceIndex.Search(request) + result := Result[xmlmodels.Piece]{Count: pieceResults.Total} + for _, hit := range pieceResults.Hits { + piece := kgpz.Pieces.Item(hit.ID) + if piece != nil { + result.Items = append(result.Items, *piece) + } + } + sw.Pieces = result + wg.Done() + }() + } + + if errissue == nil { + wg.Add(1) + go func() { + issueResults, _ := issueIndex.Search(request) + result := Result[xmlmodels.Issue]{Count: issueResults.Total} + for _, hit := range issueResults.Hits { + issue := kgpz.Issues.Item(hit.ID) + if issue != nil { + result.Items = append(result.Items, *issue) + } + } + sw.Issues = result + wg.Done() + }() + } + + wg.Wait() + + return &sw, nil +} diff --git a/views/layouts/components/_menu.gohtml b/views/layouts/components/_menu.gohtml index 2d5cb7a..49270c3 100644 --- a/views/layouts/components/_menu.gohtml +++ b/views/layouts/components/_menu.gohtml @@ -1,6 +1,15 @@
- +
@@ -28,3 +37,15 @@
+ + diff --git a/views/routes/ausgabe/body.gohtml b/views/routes/ausgabe/body.gohtml index b88ade4..f0c100e 100644 --- a/views/routes/ausgabe/body.gohtml +++ b/views/routes/ausgabe/body.gohtml @@ -1,7 +1,7 @@ {{ $model := .model }} {{ $date := .model.Datum.When }}
- + Zum Jahr {{ $date.Year }} diff --git a/views/routes/body.gohtml b/views/routes/body.gohtml index afea78b..2e61738 100644 --- a/views/routes/body.gohtml +++ b/views/routes/body.gohtml @@ -5,7 +5,7 @@
{{ range $year := .model.AvailableYears }} {{ $year }} + {{ range $i, $agent := $model.Agents.Items }} +
+ {{ $agent.String }} +
+ {{ end }} + + {{ range $i, $work := $model.Works.Items }} +
+ {{ $work.String }} +
+ {{ end }} + + {{ range $i, $place := $model.Places.Items }} +
+ {{ $place.String }} +
+ {{ end }} +
diff --git a/xmlmodels/agents.go b/xmlmodels/agents.go index 91d7c59..37eec64 100644 --- a/xmlmodels/agents.go +++ b/xmlmodels/agents.go @@ -21,10 +21,6 @@ type Agent struct { AnnotationNote } -func (a Agent) Name() string { - return "agent" -} - func (a Agent) String() string { data, _ := json.MarshalIndent(a, "", " ") return string(data) diff --git a/xmlmodels/categories.go b/xmlmodels/categories.go index 915b3be..e3cfce6 100644 --- a/xmlmodels/categories.go +++ b/xmlmodels/categories.go @@ -18,10 +18,6 @@ type Category struct { AnnotationNote } -func (c Category) Name() string { - return "category" -} - func (c Category) String() string { data, _ := json.MarshalIndent(c, "", " ") return string(data) diff --git a/xmlmodels/issues.go b/xmlmodels/issues.go index 5fcb8bb..54e82c1 100644 --- a/xmlmodels/issues.go +++ b/xmlmodels/issues.go @@ -33,10 +33,6 @@ type Additional struct { Bis int `xml:"bis"` } -func (i Issue) Name() string { - return "issue" -} - func (i Issue) Keys() []string { res := make([]string, 0, 2) res = append(res, i.Reference()) @@ -67,7 +63,7 @@ func (i Issue) Readable(_ *Library) map[string]interface{} { "ID": i.ID, "Number": i.Number.No, "Year": i.Datum.When.Year, - "Date": i.Datum.When.String(), + "Date": strconv.Itoa(i.Datum.When.Day) + "." + strconv.Itoa(i.Datum.When.Month) + "." + strconv.Itoa(i.Datum.When.Year), } for k, v := range i.AnnotationNote.Readable() { diff --git a/xmlmodels/pieces.go b/xmlmodels/pieces.go index 86d346a..0642cdc 100644 --- a/xmlmodels/pieces.go +++ b/xmlmodels/pieces.go @@ -11,7 +11,7 @@ import ( ) const ( - PIECES_CATEGORY = "piece" + PIECE_TYPE = "piece" ) type Piece struct { @@ -29,10 +29,6 @@ type Piece struct { AnnotationNote } -func (p Piece) Name() string { - return "piece" -} - func (p Piece) String() string { data, _ := json.MarshalIndent(p, "", " ") return string(data) @@ -79,7 +75,7 @@ func (p Piece) References() xmlprovider.ResolvingMap[Piece] { for _, ref := range p.CategoryRefs { if ref.Category != "" { - refs[x.Name()] = append(refs[x.Name()], xmlprovider.Resolved[Piece]{ + refs[x.Type()] = append(refs[x.Type()], xmlprovider.Resolved[Piece]{ Item: &p, Reference: ref.Category, Cert: !ref.Unsicher, @@ -94,7 +90,7 @@ func (p Piece) References() xmlprovider.ResolvingMap[Piece] { continue } if ref.Category != "" { - refs[x.Name()] = append(refs[x.Name()], xmlprovider.Resolved[Piece]{ + refs[x.Type()] = append(refs[x.Type()], xmlprovider.Resolved[Piece]{ Item: &p, Reference: ref.Category, Cert: !ref.Unsicher, @@ -102,7 +98,7 @@ func (p Piece) References() xmlprovider.ResolvingMap[Piece] { Comment: ref.Inner.InnerXML, }) } - refs[ref.Name()] = append(refs[ref.Name()], xmlprovider.Resolved[Piece]{ + refs[ref.Type()] = append(refs[ref.Type()], xmlprovider.Resolved[Piece]{ Item: &p, Reference: strconv.Itoa(ref.When.Year) + "-" + strconv.Itoa(ref.Nr), Category: ref.Category, @@ -115,7 +111,7 @@ func (p Piece) References() xmlprovider.ResolvingMap[Piece] { for _, ref := range p.PlaceRefs { if ref.Category != "" { - refs[x.Name()] = append(refs[x.Name()], xmlprovider.Resolved[Piece]{ + refs[x.Type()] = append(refs[x.Type()], xmlprovider.Resolved[Piece]{ Item: &p, Reference: ref.Category, Cert: !ref.Unsicher, @@ -123,7 +119,7 @@ func (p Piece) References() xmlprovider.ResolvingMap[Piece] { Comment: ref.Inner.InnerXML, }) } - refs[ref.Name()] = append(refs[ref.Name()], xmlprovider.Resolved[Piece]{ + refs[ref.Type()] = append(refs[ref.Type()], xmlprovider.Resolved[Piece]{ Item: &p, Reference: ref.Ref, Category: ref.Category, @@ -136,7 +132,7 @@ func (p Piece) References() xmlprovider.ResolvingMap[Piece] { for _, ref := range p.AgentRefs { if ref.Category != "" { - refs[x.Name()] = append(refs[x.Name()], xmlprovider.Resolved[Piece]{ + refs[x.Type()] = append(refs[x.Type()], xmlprovider.Resolved[Piece]{ Item: &p, Reference: ref.Category, Cert: !ref.Unsicher, @@ -144,7 +140,7 @@ func (p Piece) References() xmlprovider.ResolvingMap[Piece] { Comment: ref.Inner.InnerXML, }) } - refs[ref.Name()] = append(refs[ref.Name()], xmlprovider.Resolved[Piece]{ + refs[ref.Type()] = append(refs[ref.Type()], xmlprovider.Resolved[Piece]{ Item: &p, Reference: ref.Ref, Category: ref.Category, @@ -157,7 +153,7 @@ func (p Piece) References() xmlprovider.ResolvingMap[Piece] { for _, ref := range p.WorkRefs { if ref.Category != "" { - refs[x.Name()] = append(refs[x.Name()], xmlprovider.Resolved[Piece]{ + refs[x.Type()] = append(refs[x.Type()], xmlprovider.Resolved[Piece]{ Item: &p, Reference: ref.Category, Cert: !ref.Unsicher, @@ -165,7 +161,7 @@ func (p Piece) References() xmlprovider.ResolvingMap[Piece] { Comment: ref.Inner.InnerXML, }) } - refs[ref.Name()] = append(refs[ref.Name()], xmlprovider.Resolved[Piece]{ + refs[ref.Type()] = append(refs[ref.Type()], xmlprovider.Resolved[Piece]{ Item: &p, Reference: ref.Ref, Category: ref.Category, @@ -177,7 +173,7 @@ func (p Piece) References() xmlprovider.ResolvingMap[Piece] { for _, ref := range p.PieceRefs { if ref.Category != "" { - refs[x.Name()] = append(refs[x.Name()], xmlprovider.Resolved[Piece]{ + refs[x.Type()] = append(refs[x.Type()], xmlprovider.Resolved[Piece]{ Item: &p, Reference: ref.Category, Cert: !ref.Unsicher, @@ -186,7 +182,7 @@ func (p Piece) References() xmlprovider.ResolvingMap[Piece] { MetaData: map[string]string{}, }) } - refs[ref.Name()] = append(refs[ref.Name()], xmlprovider.Resolved[Piece]{ + refs[ref.Type()] = append(refs[ref.Type()], xmlprovider.Resolved[Piece]{ Item: &p, Reference: ref.Ref, Category: ref.Category, @@ -261,5 +257,5 @@ func (p Piece) Readable(lib *Library) map[string]interface{} { } func (p Piece) Type() string { - return PIECES_CATEGORY + return PIECE_TYPE } diff --git a/xmlmodels/places.go b/xmlmodels/places.go index d792809..5674d30 100644 --- a/xmlmodels/places.go +++ b/xmlmodels/places.go @@ -18,10 +18,6 @@ type Place struct { AnnotationNote } -func (p Place) Name() string { - return "place" -} - func (p Place) String() string { data, _ := json.MarshalIndent(p, "", " ") return string(data) diff --git a/xmlmodels/references.go b/xmlmodels/references.go index 159b3d3..1d783c8 100644 --- a/xmlmodels/references.go +++ b/xmlmodels/references.go @@ -10,9 +10,8 @@ type AgentRef struct { Reference } -func (ar AgentRef) Name() string { - var x Agent - return x.Name() +func (ar AgentRef) Type() string { + return AGENT_TYPE } func (ar AgentRef) Readable(lib *Library) map[string]interface{} { @@ -49,7 +48,7 @@ func (ir IssueRef) Readable(lib *Library) map[string]interface{} { issuekey := strconv.Itoa(ir.When.Year) + "-" + strconv.Itoa(ir.Nr) issue := lib.Issues.Item(issuekey) if issue != nil { - data["IssueDate"] = issue.Datum.When.String() + data["IssueDate"] = strconv.Itoa(issue.Datum.When.Day) + "." + strconv.Itoa(issue.Datum.When.Month) + "." + strconv.Itoa(issue.Datum.When.Year) } data["IssueNumber"] = ir.Nr @@ -57,9 +56,8 @@ func (ir IssueRef) Readable(lib *Library) map[string]interface{} { return data } -func (ir IssueRef) Name() string { - var x Issue - return x.Name() +func (ir IssueRef) Type() string { + return ISSUE_TYPE } type PlaceRef struct { @@ -80,9 +78,8 @@ func (pr *PlaceRef) Readable(lib *Library) map[string]interface{} { return data } -func (pr PlaceRef) Name() string { - var x Place - return x.Name() +func (pr PlaceRef) Type() string { + return PLACE_TYPE } type CategoryRef struct { @@ -90,9 +87,8 @@ type CategoryRef struct { Reference } -func (cr CategoryRef) Name() string { - var x Category - return x.Name() +func (cr CategoryRef) Type() string { + return CATEGORY_TYPE } func (cr CategoryRef) Readable(lib *Library) map[string]interface{} { @@ -134,9 +130,8 @@ func (wr WorkRef) Readable(lib *Library) map[string]interface{} { return data } -func (wr WorkRef) Name() string { - var x Work - return x.Name() +func (wr WorkRef) Type() string { + return WORK_TYPE } type PieceRef struct { @@ -145,7 +140,6 @@ type PieceRef struct { Reference } -func (pr PieceRef) Name() string { - var x Piece - return x.Name() +func (pr PieceRef) Type() string { + return PIECE_TYPE } diff --git a/xmlmodels/works.go b/xmlmodels/works.go index 3c8efae..13cf87d 100644 --- a/xmlmodels/works.go +++ b/xmlmodels/works.go @@ -8,7 +8,7 @@ import ( ) const ( - WORKS_CATEGORY = "work" + WORK_TYPE = "work" ) type Work struct { @@ -21,10 +21,6 @@ type Work struct { AnnotationNote } -func (w Work) Name() string { - return "work" -} - type Citation struct { XMLName xml.Name `xml:"zitation"` Title string `xml:"title"` @@ -37,7 +33,7 @@ func (w Work) References() xmlprovider.ResolvingMap[Work] { refs := make(xmlprovider.ResolvingMap[Work]) for _, ref := range w.AgentRefs { - refs[ref.Name()] = append(refs[ref.Name()], xmlprovider.Resolved[Work]{ + refs[ref.Type()] = append(refs[ref.Type()], xmlprovider.Resolved[Work]{ Item: &w, // Reference to the current Work item Reference: ref.Ref, // Reference ID Category: ref.Category, // Category of the reference @@ -78,5 +74,5 @@ func (w Work) Readable(lib *Library) map[string]interface{} { } func (w Work) Type() string { - return WORKS_CATEGORY + return WORK_TYPE }