Suche: HTMX + Webseite

This commit is contained in:
Simon Martens
2025-02-18 00:33:30 +01:00
parent fd2fa157b2
commit 7aac147686
18 changed files with 348 additions and 69 deletions

View File

@@ -27,10 +27,11 @@ const (
PRIVACY_URL = "/datenschutz/"
CONTACT_URL = "/kontakt/"
CITATION_URL = "/zitation/"
SEARCH_URL = "/suche/"
INDEX_URL = "/1764"
INDEX_URL = "/jahrgang/1764"
YEAR_OVERVIEW_URL = "/:year"
YEAR_OVERVIEW_URL = "/jahrgang/:year"
PLACE_OVERVIEW_URL = "/ort/:place"
AGENTS_OVERVIEW_URL = "/akteure/:letterorid"
CATEGORY_OVERVIEW_URL = "/kategorie/:category"
@@ -109,7 +110,13 @@ func (k *KGPZ) Init() error {
k.Enrich()
go k.Pull()
k.BuildSearchIndex()
err := k.Search.LoadIndeces()
if err != nil {
logging.Error(err, "Error loading search indeces.")
k.BuildSearchIndex()
} else {
logging.Info("Search indeces loaded.")
}
return nil
}
@@ -125,6 +132,7 @@ func (k *KGPZ) Routes(srv *fiber.App) error {
return nil
})
srv.Get(SEARCH_URL, controllers.GetSearch(k.Library, k.Search))
srv.Get(PLACE_OVERVIEW_URL, controllers.GetPlace(k.Library))
srv.Get(CATEGORY_OVERVIEW_URL, controllers.GetCategory(k.Library))
srv.Get(AGENTS_OVERVIEW_URL, controllers.GetAgents(k.Library))

26
controllers/search.go Normal file
View File

@@ -0,0 +1,26 @@
package controllers
import (
"github.com/Theodor-Springmann-Stiftung/kgpz_web/helpers/logging"
searchprovider "github.com/Theodor-Springmann-Stiftung/kgpz_web/providers/search"
"github.com/Theodor-Springmann-Stiftung/kgpz_web/viewmodels"
"github.com/Theodor-Springmann-Stiftung/kgpz_web/xmlmodels"
"github.com/gofiber/fiber/v2"
)
func GetSearch(kgpz *xmlmodels.Library, sp *searchprovider.SearchProvider) fiber.Handler {
return func(c *fiber.Ctx) error {
search := c.Query("q")
if search == "" {
return c.SendStatus(fiber.StatusNotFound)
}
view, err := viewmodels.NewSearchView(search, kgpz, sp)
if err != nil {
logging.Error(err)
return c.SendStatus(fiber.StatusNotFound)
}
return c.Render("/search/", fiber.Map{"model": view, "search": search})
}
}

View File

@@ -0,0 +1,45 @@
package datatypes
import (
"regexp"
"strings"
"unicode"
)
var html_regexp = regexp.MustCompile(`<[^>]+>`)
var ws_regexp = regexp.MustCompile(`\s+`)
func DeleteTags(s string) string {
return html_regexp.ReplaceAllString(s, "")
}
func NormalizeString(s string) string {
s = strings.TrimSpace(s)
s = strings.ReplaceAll(s, "<div>", "")
s = strings.ReplaceAll(s, "</div>", "")
return s
}
func SliceJoin[T any](slice []T, join string, f func(T) string) string {
var result []string
for _, item := range slice {
ap := f(item)
if ap != "" {
result = append(result, ap)
}
}
return strings.Join(result, join)
}
func RemovePunctuation(s string) string {
return strings.Map(func(r rune) rune {
if unicode.IsPunct(r) {
return -1
}
return r
}, s)
}
func NormalizeWhitespace(s string) string {
return strings.TrimSpace(ws_regexp.ReplaceAllString(s, " "))
}

View File

@@ -50,7 +50,31 @@ func (sp *SearchProvider) Index(item ISearchable, lib *xmlmodels.Library) error
return err
}
return i.Index(keys[0], item.Readable(lib))
read := item.Readable(lib)
return i.Index(keys[0], read)
}
// TODO: this is sloppy
func (sp *SearchProvider) LoadIndeces() error {
files, err := filepath.Glob(filepath.Join(sp.basepath, "*.bleve"))
if err != nil {
return err
}
if len(files) == 0 {
return errors.New("No indeces found.")
}
for _, file := range files {
index, err := bleve.Open(file)
if err != nil {
return err
}
typ := filepath.Base(file)
typ = typ[:len(typ)-6]
sp.indeces.Store(typ, index)
}
return nil
}
func (sp *SearchProvider) FindCreateIndex(typ string) (bleve.Index, error) {
@@ -77,6 +101,16 @@ func (sp *SearchProvider) FindCreateIndex(typ string) (bleve.Index, error) {
return ind, nil
}
func (sp *SearchProvider) GetIndex(typ string) (bleve.Index, error) {
index, ok := sp.indeces.Load(typ)
if !ok {
return nil, errors.New("Index not found.")
}
i := index.(bleve.Index)
return i, nil
}
func default_mapping() (*mapping.IndexMappingImpl, error) {
indexMapping := bleve.NewIndexMapping()

View File

@@ -8,7 +8,7 @@ type IXMLItem interface {
// - Keys should be unique
// - Keys[0] has the special meaning of the primary key (for FTS etc.)
Keys() []string
Name() string
Type() string
}
type ILibrary interface {

View File

@@ -147,7 +147,7 @@ func (p *XMLProvider[T]) ReverseLookup(item IXMLItem) []Resolved[T] {
keys := item.Keys()
for _, key := range keys {
r, err := p.Resolver.Get(item.Name(), key)
r, err := p.Resolver.Get(item.Type(), key)
if err == nil {
ret = append(ret, r...)
}

153
viewmodels/searchview.go Normal file
View File

@@ -0,0 +1,153 @@
package viewmodels
import (
"fmt"
"sync"
"github.com/Theodor-Springmann-Stiftung/kgpz_web/helpers/datatypes"
searchprovider "github.com/Theodor-Springmann-Stiftung/kgpz_web/providers/search"
"github.com/Theodor-Springmann-Stiftung/kgpz_web/xmlmodels"
"github.com/blevesearch/bleve/v2"
"golang.org/x/text/cases"
"golang.org/x/text/language"
"golang.org/x/text/unicode/norm"
)
type Result[T any] struct {
Count uint64
Items []T
}
type SearchView struct {
Agents Result[xmlmodels.Agent]
Works Result[xmlmodels.Work]
Places Result[xmlmodels.Place]
Categories Result[xmlmodels.Category]
Pieces Result[xmlmodels.Piece]
Issues Result[xmlmodels.Issue]
}
func NewSearchView(search string, kgpz *xmlmodels.Library, sp *searchprovider.SearchProvider) (*SearchView, error) {
sw := SearchView{}
search = datatypes.DeleteTags(search)
search = datatypes.NormalizeString(search)
search = datatypes.RemovePunctuation(search)
search = cases.Lower(language.German).String(search)
search = norm.NFKD.String(search)
query := bleve.NewTermQuery(search)
request := bleve.NewSearchRequest(query)
request.Size = 100
agentIndex, erragent := sp.GetIndex(xmlmodels.AGENT_TYPE)
workIndex, errwork := sp.GetIndex(xmlmodels.WORK_TYPE)
placeIndex, errplace := sp.GetIndex(xmlmodels.PLACE_TYPE)
categoryIndex, errcategory := sp.GetIndex(xmlmodels.CATEGORY_TYPE)
pieceIndex, errpiece := sp.GetIndex(xmlmodels.PIECE_TYPE)
issueIndex, errissue := sp.GetIndex(xmlmodels.ISSUE_TYPE)
if agentIndex == nil || workIndex == nil || placeIndex == nil || categoryIndex == nil || pieceIndex == nil || issueIndex == nil {
return nil, fmt.Errorf("Indeces not found.")
}
wg := sync.WaitGroup{}
if erragent == nil {
wg.Add(1)
go func() {
agentResults, _ := agentIndex.Search(request)
result := Result[xmlmodels.Agent]{Count: agentResults.Total}
for _, hit := range agentResults.Hits {
agent := kgpz.Agents.Item(hit.ID)
if agent != nil {
result.Items = append(result.Items, *agent)
}
}
sw.Agents = result
wg.Done()
}()
}
if errwork == nil {
wg.Add(1)
go func() {
workResults, _ := workIndex.Search(request)
result := Result[xmlmodels.Work]{Count: workResults.Total}
for _, hit := range workResults.Hits {
work := kgpz.Works.Item(hit.ID)
if work != nil {
result.Items = append(result.Items, *work)
}
}
sw.Works = result
wg.Done()
}()
}
if errplace == nil {
wg.Add(1)
go func() {
placeResults, _ := placeIndex.Search(request)
result := Result[xmlmodels.Place]{Count: placeResults.Total}
for _, hit := range placeResults.Hits {
place := kgpz.Places.Item(hit.ID)
if place != nil {
result.Items = append(result.Items, *place)
}
}
sw.Places = result
wg.Done()
}()
}
if errcategory == nil {
wg.Add(1)
go func() {
categoryResults, _ := categoryIndex.Search(request)
result := Result[xmlmodels.Category]{Count: categoryResults.Total}
for _, hit := range categoryResults.Hits {
category := kgpz.Categories.Item(hit.ID)
if category != nil {
result.Items = append(result.Items, *category)
}
}
sw.Categories = result
wg.Done()
}()
}
if errpiece == nil {
wg.Add(1)
go func() {
pieceResults, _ := pieceIndex.Search(request)
result := Result[xmlmodels.Piece]{Count: pieceResults.Total}
for _, hit := range pieceResults.Hits {
piece := kgpz.Pieces.Item(hit.ID)
if piece != nil {
result.Items = append(result.Items, *piece)
}
}
sw.Pieces = result
wg.Done()
}()
}
if errissue == nil {
wg.Add(1)
go func() {
issueResults, _ := issueIndex.Search(request)
result := Result[xmlmodels.Issue]{Count: issueResults.Total}
for _, hit := range issueResults.Hits {
issue := kgpz.Issues.Item(hit.ID)
if issue != nil {
result.Items = append(result.Items, *issue)
}
}
sw.Issues = result
wg.Done()
}()
}
wg.Wait()
return &sw, nil
}

View File

@@ -1,6 +1,15 @@
<div class="flex flex-row justify-center mt-8">
<div class="w-6/12">
<input type="search" placeholder="Suche" class="px-2.5 py-1.5 border w-full" />
<input
type="search"
name="q"
id="search"
placeholder="Suche"
class="px-2.5 py-1.5 border w-full"
hx-get="/suche/?noCache=true"
hx-trigger="input changed delay:200ms, keyup[key=='Enter']"
hx-select="main"
hx-target="main" />
</div>
<div x-data="{ open: false }">
@@ -28,3 +37,15 @@
</div>
</div>
</div>
<script>
document.body.addEventListener("htmx:configRequest", function (event) {
console.log("Before request event triggered");
let t = event.detail.elt; // Get the element triggering the request
if (t.id === "search" && t.value === "") {
event.detail.parameters = {};
event.detail.path = window.location.pathname + window.location.search;
}
});
</script>

View File

@@ -1,7 +1,7 @@
{{ $model := .model }}
{{ $date := .model.Datum.When }}
<div>
<a href="/{{- $date.Year -}}">
<a href="/jahrgang/{{- $date.Year -}}">
Zum Jahr
{{ $date.Year }}
</a>

View File

@@ -5,7 +5,7 @@
<div class="mx-auto flex flex-row gap-x-4 w-fit items-end leading-none">
{{ range $year := .model.AvailableYears }}
<a
href="/{{ $year }}"
href="/jahrgang/{{ $year }}"
class="no-underline leading-none !m-0 !p-0
{{ if eq $year $y }}text-2xl font-bold pointer-events-none" aria-current="page{{ end }}"
>{{ $year }}</a

View File

@@ -0,0 +1,22 @@
{{ $model := .model }}
<div id="results">
{{ range $i, $agent := $model.Agents.Items }}
<div>
{{ $agent.String }}
</div>
{{ end }}
{{ range $i, $work := $model.Works.Items }}
<div>
{{ $work.String }}
</div>
{{ end }}
{{ range $i, $place := $model.Places.Items }}
<div>
{{ $place.String }}
</div>
{{ end }}
</div>

View File

@@ -21,10 +21,6 @@ type Agent struct {
AnnotationNote
}
func (a Agent) Name() string {
return "agent"
}
func (a Agent) String() string {
data, _ := json.MarshalIndent(a, "", " ")
return string(data)

View File

@@ -18,10 +18,6 @@ type Category struct {
AnnotationNote
}
func (c Category) Name() string {
return "category"
}
func (c Category) String() string {
data, _ := json.MarshalIndent(c, "", " ")
return string(data)

View File

@@ -33,10 +33,6 @@ type Additional struct {
Bis int `xml:"bis"`
}
func (i Issue) Name() string {
return "issue"
}
func (i Issue) Keys() []string {
res := make([]string, 0, 2)
res = append(res, i.Reference())
@@ -67,7 +63,7 @@ func (i Issue) Readable(_ *Library) map[string]interface{} {
"ID": i.ID,
"Number": i.Number.No,
"Year": i.Datum.When.Year,
"Date": i.Datum.When.String(),
"Date": strconv.Itoa(i.Datum.When.Day) + "." + strconv.Itoa(i.Datum.When.Month) + "." + strconv.Itoa(i.Datum.When.Year),
}
for k, v := range i.AnnotationNote.Readable() {

View File

@@ -11,7 +11,7 @@ import (
)
const (
PIECES_CATEGORY = "piece"
PIECE_TYPE = "piece"
)
type Piece struct {
@@ -29,10 +29,6 @@ type Piece struct {
AnnotationNote
}
func (p Piece) Name() string {
return "piece"
}
func (p Piece) String() string {
data, _ := json.MarshalIndent(p, "", " ")
return string(data)
@@ -79,7 +75,7 @@ func (p Piece) References() xmlprovider.ResolvingMap[Piece] {
for _, ref := range p.CategoryRefs {
if ref.Category != "" {
refs[x.Name()] = append(refs[x.Name()], xmlprovider.Resolved[Piece]{
refs[x.Type()] = append(refs[x.Type()], xmlprovider.Resolved[Piece]{
Item: &p,
Reference: ref.Category,
Cert: !ref.Unsicher,
@@ -94,7 +90,7 @@ func (p Piece) References() xmlprovider.ResolvingMap[Piece] {
continue
}
if ref.Category != "" {
refs[x.Name()] = append(refs[x.Name()], xmlprovider.Resolved[Piece]{
refs[x.Type()] = append(refs[x.Type()], xmlprovider.Resolved[Piece]{
Item: &p,
Reference: ref.Category,
Cert: !ref.Unsicher,
@@ -102,7 +98,7 @@ func (p Piece) References() xmlprovider.ResolvingMap[Piece] {
Comment: ref.Inner.InnerXML,
})
}
refs[ref.Name()] = append(refs[ref.Name()], xmlprovider.Resolved[Piece]{
refs[ref.Type()] = append(refs[ref.Type()], xmlprovider.Resolved[Piece]{
Item: &p,
Reference: strconv.Itoa(ref.When.Year) + "-" + strconv.Itoa(ref.Nr),
Category: ref.Category,
@@ -115,7 +111,7 @@ func (p Piece) References() xmlprovider.ResolvingMap[Piece] {
for _, ref := range p.PlaceRefs {
if ref.Category != "" {
refs[x.Name()] = append(refs[x.Name()], xmlprovider.Resolved[Piece]{
refs[x.Type()] = append(refs[x.Type()], xmlprovider.Resolved[Piece]{
Item: &p,
Reference: ref.Category,
Cert: !ref.Unsicher,
@@ -123,7 +119,7 @@ func (p Piece) References() xmlprovider.ResolvingMap[Piece] {
Comment: ref.Inner.InnerXML,
})
}
refs[ref.Name()] = append(refs[ref.Name()], xmlprovider.Resolved[Piece]{
refs[ref.Type()] = append(refs[ref.Type()], xmlprovider.Resolved[Piece]{
Item: &p,
Reference: ref.Ref,
Category: ref.Category,
@@ -136,7 +132,7 @@ func (p Piece) References() xmlprovider.ResolvingMap[Piece] {
for _, ref := range p.AgentRefs {
if ref.Category != "" {
refs[x.Name()] = append(refs[x.Name()], xmlprovider.Resolved[Piece]{
refs[x.Type()] = append(refs[x.Type()], xmlprovider.Resolved[Piece]{
Item: &p,
Reference: ref.Category,
Cert: !ref.Unsicher,
@@ -144,7 +140,7 @@ func (p Piece) References() xmlprovider.ResolvingMap[Piece] {
Comment: ref.Inner.InnerXML,
})
}
refs[ref.Name()] = append(refs[ref.Name()], xmlprovider.Resolved[Piece]{
refs[ref.Type()] = append(refs[ref.Type()], xmlprovider.Resolved[Piece]{
Item: &p,
Reference: ref.Ref,
Category: ref.Category,
@@ -157,7 +153,7 @@ func (p Piece) References() xmlprovider.ResolvingMap[Piece] {
for _, ref := range p.WorkRefs {
if ref.Category != "" {
refs[x.Name()] = append(refs[x.Name()], xmlprovider.Resolved[Piece]{
refs[x.Type()] = append(refs[x.Type()], xmlprovider.Resolved[Piece]{
Item: &p,
Reference: ref.Category,
Cert: !ref.Unsicher,
@@ -165,7 +161,7 @@ func (p Piece) References() xmlprovider.ResolvingMap[Piece] {
Comment: ref.Inner.InnerXML,
})
}
refs[ref.Name()] = append(refs[ref.Name()], xmlprovider.Resolved[Piece]{
refs[ref.Type()] = append(refs[ref.Type()], xmlprovider.Resolved[Piece]{
Item: &p,
Reference: ref.Ref,
Category: ref.Category,
@@ -177,7 +173,7 @@ func (p Piece) References() xmlprovider.ResolvingMap[Piece] {
for _, ref := range p.PieceRefs {
if ref.Category != "" {
refs[x.Name()] = append(refs[x.Name()], xmlprovider.Resolved[Piece]{
refs[x.Type()] = append(refs[x.Type()], xmlprovider.Resolved[Piece]{
Item: &p,
Reference: ref.Category,
Cert: !ref.Unsicher,
@@ -186,7 +182,7 @@ func (p Piece) References() xmlprovider.ResolvingMap[Piece] {
MetaData: map[string]string{},
})
}
refs[ref.Name()] = append(refs[ref.Name()], xmlprovider.Resolved[Piece]{
refs[ref.Type()] = append(refs[ref.Type()], xmlprovider.Resolved[Piece]{
Item: &p,
Reference: ref.Ref,
Category: ref.Category,
@@ -261,5 +257,5 @@ func (p Piece) Readable(lib *Library) map[string]interface{} {
}
func (p Piece) Type() string {
return PIECES_CATEGORY
return PIECE_TYPE
}

View File

@@ -18,10 +18,6 @@ type Place struct {
AnnotationNote
}
func (p Place) Name() string {
return "place"
}
func (p Place) String() string {
data, _ := json.MarshalIndent(p, "", " ")
return string(data)

View File

@@ -10,9 +10,8 @@ type AgentRef struct {
Reference
}
func (ar AgentRef) Name() string {
var x Agent
return x.Name()
func (ar AgentRef) Type() string {
return AGENT_TYPE
}
func (ar AgentRef) Readable(lib *Library) map[string]interface{} {
@@ -49,7 +48,7 @@ func (ir IssueRef) Readable(lib *Library) map[string]interface{} {
issuekey := strconv.Itoa(ir.When.Year) + "-" + strconv.Itoa(ir.Nr)
issue := lib.Issues.Item(issuekey)
if issue != nil {
data["IssueDate"] = issue.Datum.When.String()
data["IssueDate"] = strconv.Itoa(issue.Datum.When.Day) + "." + strconv.Itoa(issue.Datum.When.Month) + "." + strconv.Itoa(issue.Datum.When.Year)
}
data["IssueNumber"] = ir.Nr
@@ -57,9 +56,8 @@ func (ir IssueRef) Readable(lib *Library) map[string]interface{} {
return data
}
func (ir IssueRef) Name() string {
var x Issue
return x.Name()
func (ir IssueRef) Type() string {
return ISSUE_TYPE
}
type PlaceRef struct {
@@ -80,9 +78,8 @@ func (pr *PlaceRef) Readable(lib *Library) map[string]interface{} {
return data
}
func (pr PlaceRef) Name() string {
var x Place
return x.Name()
func (pr PlaceRef) Type() string {
return PLACE_TYPE
}
type CategoryRef struct {
@@ -90,9 +87,8 @@ type CategoryRef struct {
Reference
}
func (cr CategoryRef) Name() string {
var x Category
return x.Name()
func (cr CategoryRef) Type() string {
return CATEGORY_TYPE
}
func (cr CategoryRef) Readable(lib *Library) map[string]interface{} {
@@ -134,9 +130,8 @@ func (wr WorkRef) Readable(lib *Library) map[string]interface{} {
return data
}
func (wr WorkRef) Name() string {
var x Work
return x.Name()
func (wr WorkRef) Type() string {
return WORK_TYPE
}
type PieceRef struct {
@@ -145,7 +140,6 @@ type PieceRef struct {
Reference
}
func (pr PieceRef) Name() string {
var x Piece
return x.Name()
func (pr PieceRef) Type() string {
return PIECE_TYPE
}

View File

@@ -8,7 +8,7 @@ import (
)
const (
WORKS_CATEGORY = "work"
WORK_TYPE = "work"
)
type Work struct {
@@ -21,10 +21,6 @@ type Work struct {
AnnotationNote
}
func (w Work) Name() string {
return "work"
}
type Citation struct {
XMLName xml.Name `xml:"zitation"`
Title string `xml:"title"`
@@ -37,7 +33,7 @@ func (w Work) References() xmlprovider.ResolvingMap[Work] {
refs := make(xmlprovider.ResolvingMap[Work])
for _, ref := range w.AgentRefs {
refs[ref.Name()] = append(refs[ref.Name()], xmlprovider.Resolved[Work]{
refs[ref.Type()] = append(refs[ref.Type()], xmlprovider.Resolved[Work]{
Item: &w, // Reference to the current Work item
Reference: ref.Ref, // Reference ID
Category: ref.Category, // Category of the reference
@@ -78,5 +74,5 @@ func (w Work) Readable(lib *Library) map[string]interface{} {
}
func (w Work) Type() string {
return WORKS_CATEGORY
return WORK_TYPE
}