diff --git a/dbmodels/agents.go b/dbmodels/agents.go index cefd053..bdcf1e3 100644 --- a/dbmodels/agents.go +++ b/dbmodels/agents.go @@ -26,14 +26,13 @@ func AgentForId(app core.App, id string) (*Agent, error) { func FTS5SearchAgents(app core.App, query string) ([]*Agent, error) { a := []*Agent{} q := NormalizeQuery(query) - if len(q) == 0 { + req := IntoQueryRequests([]string{AGENTS_NAME_FIELD, AGENTS_PSEUDONYMS_FIELD, REFERENCES_FIELD, AGENTS_BIOGRAPHICAL_DATA_FIELD, ANNOTATION_FIELD}, q) + + if len(req) == 0 { return a, nil } - ids, err := FTS5Search(app, AGENTS_TABLE, FTS5QueryRequest{ - Fields: []string{AGENTS_NAME_FIELD, AGENTS_PSEUDONYMS_FIELD, REFERENCES_FIELD, AGENTS_BIOGRAPHICAL_DATA_FIELD, ANNOTATION_FIELD}, - Query: q, - }) + ids, err := FTS5Search(app, AGENTS_TABLE, req...) if err != nil { return nil, err diff --git a/dbmodels/fts5.go b/dbmodels/fts5.go index f329651..65c6f10 100644 --- a/dbmodels/fts5.go +++ b/dbmodels/fts5.go @@ -2,8 +2,10 @@ package dbmodels import ( "errors" + "fmt" "strconv" "strings" + "unicode" "github.com/Theodor-Springmann-Stiftung/musenalm/helpers/datatypes" "github.com/pocketbase/dbx" @@ -99,22 +101,125 @@ var CONTENTS_FTS5_FIELDS = []string{ var ErrInvalidQuery = errors.New("invalid input into the search function") -func NormalizeQuery(query string) []string { - query = datatypes.NormalizeString(query) - query = datatypes.DeleteTags(query) - query = datatypes.RemovePunctuation(query) - query = cases.Lower(language.German).String(query) - // TODO: how to normalize, which unicode normalization to use? +type Query struct { + Include []string // Phrases that should be matched + Exclude []string // Phrases that should not be matched + UnsafeI []string // Phrases < 3 characters + UnsafeE []string // Phrases < 3 characters excluded +} - split := strings.Split(query, " ") - res := []string{} - for _, s := range split { - if len(s) > 2 { - res = append(res, s) +// Parses query strings like +// word another "this is a phrase" -notthis aword -alsonotthis -"also not this" +// into seperate phrases +func NormalizeQuery(query string) Query { + query = datatypes.NormalizeString(query) + // TODO: how to normalize, which unicode normalization to use? + // query = datatypes.RemovePunctuation(query) + query = cases.Lower(language.German).String(query) + + var include []string + var exclude []string + var unsafeI []string + var unsafeE []string + + isInQuotes := false + isExcluded := false + + var cToken strings.Builder + + at := func() { + if cToken.Len() == 0 { + return + } + + t := cToken.String() + if len(t) < 3 && isExcluded { + unsafeE = append(unsafeE, t) + return + } else if len(t) < 3 { + unsafeI = append(unsafeI, t) + return + } + + if len(t) >= 3 && isExcluded { + exclude = append(exclude, t) + return + } else if len(t) >= 3 { + include = append(include, t) + return } } - return res + reset := func() { + isInQuotes = false + isExcluded = false + cToken.Reset() + } + + addToken := func() { + at() + reset() + } + + for _, r := range query { + fmt.Printf("Rune: %v\n", r) + if r == '"' { + if isInQuotes { + addToken() + } else if cToken.Len() == 0 { + isInQuotes = true + } + // INFO: - is punctuation, so the order of cases is important + } else if r == 45 && cToken.Len() == 0 { + isExcluded = true + } else if unicode.IsSpace(r) && !isInQuotes { + addToken() + } else if unicode.IsPunct(r) && !isInQuotes { + addToken() + } else { + cToken.WriteRune(r) + } + } + + if cToken.Len() > 0 { + at() + } + + fmt.Printf("Query: %v\n", query) + fmt.Printf("Include: %v\n", include) + fmt.Printf("Exclude: %v\n", exclude) + fmt.Printf("UnsafeI: %v\n", unsafeI) + fmt.Printf("UnsafeE: %v\n", unsafeE) + + return Query{ + Include: include, + Exclude: exclude, + UnsafeI: unsafeI, + UnsafeE: unsafeE, + } +} + +// INFO: Takes in fields and a Query object +func IntoQueryRequests(f []string, q Query) []FTS5QueryRequest { + ret := []FTS5QueryRequest{} + + if len(q.Include) > 0 { + ret = append(ret, FTS5QueryRequest{ + Fields: f, + Query: q.Include, + OP: OP_AND, + }) + } + + if len(q.Exclude) > 0 { + ret = append(ret, FTS5QueryRequest{ + Fields: f, + Query: q.Exclude, + OP: OP_NOT, + }) + } + + return ret } func FTS5Search(app core.App, table string, mapfq ...FTS5QueryRequest) ([]*FTS5IDQueryResult, error) { @@ -125,7 +230,16 @@ func FTS5Search(app core.App, table string, mapfq ...FTS5QueryRequest) ([]*FTS5I q := NewFTS5Query().From(table).SelectID() for _, v := range mapfq { for _, que := range v.Query { - q.AndMatch(v.Fields, que) + switch v.OP { + case OP_AND: + q.AndMatch(v.Fields, que) + case OP_OR: + q.OrMatch(v.Fields, que) + case OP_NOT: + q.NotMatch(v.Fields, que) + case NONE: + q.AndMatch(v.Fields, que) + } } } diff --git a/dbmodels/fts5query.go b/dbmodels/fts5query.go index 32d7c27..3fa768e 100644 --- a/dbmodels/fts5query.go +++ b/dbmodels/fts5query.go @@ -9,6 +9,7 @@ import ( type FTS5QueryRequest struct { Fields []string Query []string + OP Operator } type FTS5IDQueryResult struct { diff --git a/dbmodels/queries.go b/dbmodels/queries.go index 3c6561d..b655b49 100644 --- a/dbmodels/queries.go +++ b/dbmodels/queries.go @@ -112,6 +112,11 @@ func Series_IDs(app core.App, ids []any) ([]*Series, error) { return TableByIDs[[]*Series](app, SERIES_TABLE, ids) } +func Series_MusenalmID(app core.App, id string) (*Series, error) { + ret, err := TableByField[Series](app, SERIES_TABLE, MUSENALMID_FIELD, id) + return &ret, err +} + func Series_ID(app core.App, id string) (*Series, error) { ret, err := TableByID[Series](app, SERIES_TABLE, id) return &ret, err diff --git a/dbmodels/seriesses.go b/dbmodels/seriesses.go index 759d736..b6d94e9 100644 --- a/dbmodels/seriesses.go +++ b/dbmodels/seriesses.go @@ -43,14 +43,13 @@ func BasicSearchSeries(app core.App, query string) ([]*Series, []*Series, error) // INFO: Needing to differentiate matches querysplit := NormalizeQuery(query) - if len(querysplit) == 0 { + req := IntoQueryRequests([]string{SERIES_TITLE_FIELD, ANNOTATION_FIELD, REFERENCES_FIELD}, querysplit) + + if len(req) == 0 { return series, []*Series{}, nil } - altids, err := FTS5Search(app, SERIES_TABLE, FTS5QueryRequest{ - Fields: []string{SERIES_TITLE_FIELD, ANNOTATION_FIELD, REFERENCES_FIELD}, - Query: querysplit, - }) + altids, err := FTS5Search(app, SERIES_TABLE, req...) if err != nil { return nil, nil, err } diff --git a/pages/reihe.go b/pages/reihe.go index f6f5b9e..18ff100 100644 --- a/pages/reihe.go +++ b/pages/reihe.go @@ -35,13 +35,13 @@ func (p *ReihePage) Setup(router *router.Router[*core.RequestEvent], app core.Ap router.GET(URL_REIHE, func(e *core.RequestEvent) error { id := e.Request.PathValue("id") data := make(map[string]interface{}) - reihe, err := dbmodels.Series_ID(app, id) - if err != nil { + reihe, err := dbmodels.Series_MusenalmID(app, id) + if err != nil || reihe == nil || reihe.Id == "" { return engine.Response404(e, err, data) } data["series"] = reihe - entries, relations, err := Entries_Series_IDs(app, []any{id}) + entries, relations, err := Entries_Series_IDs(app, []any{reihe.Id}) if err != nil { return engine.Response404(e, err, data) } diff --git a/pages/suche.go b/pages/suche.go index 1c143b6..2f22543 100644 --- a/pages/suche.go +++ b/pages/suche.go @@ -1,9 +1,11 @@ package pages import ( + "database/sql" "fmt" "net/http" "slices" + "strings" "github.com/Theodor-Springmann-Stiftung/musenalm/app" "github.com/Theodor-Springmann-Stiftung/musenalm/dbmodels" @@ -54,7 +56,7 @@ func (p *SuchePage) Setup(router *router.Router[*core.RequestEvent], app core.Ap allparas, _ := NewSearchParameters(e, *paras) - if paras.Query != "" || allparas.IsBaendeSearch() { + if allparas.IsBaendeSearch() { return p.SearchBaendeRequest(app, engine, e, *allparas) } @@ -140,7 +142,7 @@ func NewParameters(e *core.RequestEvent) (*Parameters, error) { }, nil } -func (p *Parameters) NormalizeQuery() []string { +func (p *Parameters) NormalizeQuery() dbmodels.Query { return dbmodels.NormalizeQuery(p.Query) } @@ -151,7 +153,6 @@ type SearchParameters struct { Annotations bool Persons bool Title bool - Alm bool Series bool Places bool Refs bool @@ -170,7 +171,6 @@ type SearchParameters struct { } func NewSearchParameters(e *core.RequestEvent, p Parameters) (*SearchParameters, error) { - alm := e.Request.URL.Query().Get(BAENDE_PARAM_ALM_NR) == "on" title := e.Request.URL.Query().Get(BAENDE_PARAM_TITLE) == "on" series := e.Request.URL.Query().Get(BAENDE_PARAM_SERIES) == "on" persons := e.Request.URL.Query().Get(BAENDE_PARAM_PERSONS) == "on" @@ -180,14 +180,19 @@ func NewSearchParameters(e *core.RequestEvent, p Parameters) (*SearchParameters, year := e.Request.URL.Query().Get(BAENDE_PARAM_YEAR) == "on" almstring := e.Request.URL.Query().Get(BAENDE_PARAM_ALM_NR + "string") + titlestring := e.Request.URL.Query().Get(BAENDE_PARAM_TITLE + "string") seriesstring := e.Request.URL.Query().Get(BAENDE_PARAM_SERIES + "string") personsstring := e.Request.URL.Query().Get(BAENDE_PARAM_PERSONS + "string") placesstring := e.Request.URL.Query().Get(BAENDE_PARAM_PLACES + "string") - refsstring := e.Request.URL.Query().Get(BAENDE_PARAM_REFS + "string") annotationsstring := e.Request.URL.Query().Get(BAENDE_PARAM_ANNOTATIONS + "string") yearstring := e.Request.URL.Query().Get(BAENDE_PARAM_YEAR + "string") + refss := e.Request.URL.Query().Get(BAENDE_PARAM_REFS + "string") + if refss != "" { + refss = "\"" + refss + "\"" + } + sort := e.Request.URL.Query().Get("sort") if sort == "" { sort = "year" @@ -197,7 +202,6 @@ func NewSearchParameters(e *core.RequestEvent, p Parameters) (*SearchParameters, Parameters: p, Sort: sort, // INFO: Common parameters - Alm: alm, Title: title, Persons: persons, Annotations: annotations, @@ -214,15 +218,36 @@ func NewSearchParameters(e *core.RequestEvent, p Parameters) (*SearchParameters, SeriesString: seriesstring, PersonsString: personsstring, PlacesString: placesstring, - RefsString: refsstring, + RefsString: refss, AnnotationsString: annotationsstring, YearString: yearstring, }, nil } func (p SearchParameters) AllSearchTerms() string { - q := p.Query + " " + p.AnnotationsString + " " + p.PersonsString + " " + p.TitleString + " " + p.AlmString + " " + p.SeriesString + " " + p.PlacesString + " " + p.RefsString + " " + p.YearString - return q + res := []string{} + res = append(res, p.includedParams(p.Query)...) + res = append(res, p.includedParams(p.AnnotationsString)...) + res = append(res, p.includedParams(p.PersonsString)...) + res = append(res, p.includedParams(p.TitleString)...) + res = append(res, p.includedParams(p.SeriesString)...) + res = append(res, p.includedParams(p.PlacesString)...) + res = append(res, p.includedParams(p.RefsString)...) + res = append(res, p.includedParams(p.YearString)...) + res = append(res, p.AlmString) + return strings.Join(res, " ") +} + +func (p SearchParameters) includedParams(q string) []string { + res := []string{} + que := dbmodels.NormalizeQuery(q) + for _, qq := range que.Include { + res = append(res, qq) + } + for _, qq := range que.UnsafeI { + res = append(res, qq) + } + return res } func (p SearchParameters) ToQueryParams() string { @@ -235,31 +260,28 @@ func (p SearchParameters) ToQueryParams() string { if p.Query != "" { q += fmt.Sprintf("q=%s", p.Query) - } - if p.Alm { - q += "&alm=on" - } - if p.Title { - q += "&title=on" - } - if p.Persons { - q += "&persons=on" - } - if p.Annotations { - q += "&annotations=on" - } - if p.Series { - q += "&series=on" - } - if p.Places { - q += "&places=on" - } - if p.Refs { - q += "&references=on" - } - if p.Year { - q += "&year=on" + if p.Title { + q += "&title=on" + } + if p.Persons { + q += "&persons=on" + } + if p.Annotations { + q += "&annotations=on" + } + if p.Series { + q += "&series=on" + } + if p.Places { + q += "&places=on" + } + if p.Refs { + q += "&references=on" + } + if p.Year { + q += "&year=on" + } } if p.YearString != "" { @@ -274,9 +296,6 @@ func (p SearchParameters) ToQueryParams() string { if p.TitleString != "" { q += fmt.Sprintf("&titlestring=%s", p.TitleString) } - if p.AlmString != "" { - q += fmt.Sprintf("&almstring=%s", p.AlmString) - } if p.SeriesString != "" { q += fmt.Sprintf("&seriesstring=%s", p.SeriesString) } @@ -291,16 +310,13 @@ func (p SearchParameters) ToQueryParams() string { } func (p SearchParameters) IsBaendeSearch() bool { - return p.Collection == "baende" && (p.Query != "" || (p.AnnotationsString != "" || p.PersonsString != "" || p.TitleString != "" || p.AlmString != "" || p.SeriesString != "" || p.PlacesString != "" || p.RefsString != "" || p.YearString != "")) + return p.Collection == "baende" && (p.Query != "" || p.AlmString != "" || p.AnnotationsString != "" || p.PersonsString != "" || p.TitleString != "" || p.SeriesString != "" || p.PlacesString != "" || p.RefsString != "" || p.YearString != "") } func (p SearchParameters) FieldSetBaende() []dbmodels.FTS5QueryRequest { ret := []dbmodels.FTS5QueryRequest{} if p.Query != "" { fields := []string{dbmodels.ID_FIELD} - if p.Alm { - fields = append(fields, dbmodels.MUSENALMID_FIELD) - } if p.Title { // INFO: Preferred Title is not here to avoid hitting the Reihentitel fields = append(fields, @@ -331,93 +347,50 @@ func (p SearchParameters) FieldSetBaende() []dbmodels.FTS5QueryRequest { } que := p.NormalizeQuery() - - if len(que) > 0 { - ret = append(ret, dbmodels.FTS5QueryRequest{ - Fields: fields, - Query: p.NormalizeQuery(), - }) - } + req := dbmodels.IntoQueryRequests(fields, que) + ret = append(ret, req...) } if p.AnnotationsString != "" { que := dbmodels.NormalizeQuery(p.AnnotationsString) - if len(que) > 0 { - ret = append(ret, dbmodels.FTS5QueryRequest{ - Fields: []string{dbmodels.ANNOTATION_FIELD}, - Query: que, - }) - } + req := dbmodels.IntoQueryRequests([]string{dbmodels.ANNOTATION_FIELD}, que) + ret = append(ret, req...) } if p.PersonsString != "" { que := dbmodels.NormalizeQuery(p.PersonsString) - if len(que) > 0 { - ret = append(ret, dbmodels.FTS5QueryRequest{ - Fields: []string{dbmodels.AGENTS_TABLE, dbmodels.RESPONSIBILITY_STMT_FIELD}, - Query: que, - }) - } + req := dbmodels.IntoQueryRequests([]string{dbmodels.AGENTS_TABLE, dbmodels.RESPONSIBILITY_STMT_FIELD}, que) + ret = append(ret, req...) } if p.TitleString != "" { que := dbmodels.NormalizeQuery(p.TitleString) - if len(que) > 0 { - ret = append(ret, dbmodels.FTS5QueryRequest{ - Fields: []string{dbmodels.TITLE_STMT_FIELD, dbmodels.SUBTITLE_STMT_FIELD, dbmodels.INCIPIT_STMT_FIELD, dbmodels.VARIANT_TITLE_FIELD, dbmodels.PARALLEL_TITLE_FIELD}, - Query: que, - }) - } - } - - if p.AlmString != "" { - que := dbmodels.NormalizeQuery(p.AlmString) - if len(que) > 0 { - ret = append(ret, dbmodels.FTS5QueryRequest{ - Fields: []string{dbmodels.MUSENALMID_FIELD}, - Query: que, - }) - } + req := dbmodels.IntoQueryRequests([]string{dbmodels.TITLE_STMT_FIELD, dbmodels.SUBTITLE_STMT_FIELD, dbmodels.INCIPIT_STMT_FIELD, dbmodels.VARIANT_TITLE_FIELD, dbmodels.PARALLEL_TITLE_FIELD}, que) + ret = append(ret, req...) } if p.SeriesString != "" { que := dbmodels.NormalizeQuery(p.SeriesString) - if len(que) > 0 { - ret = append(ret, dbmodels.FTS5QueryRequest{ - Fields: []string{dbmodels.SERIES_TABLE}, - Query: que, - }) - } + req := dbmodels.IntoQueryRequests([]string{dbmodels.SERIES_TABLE}, que) + ret = append(ret, req...) } if p.PlacesString != "" { que := dbmodels.NormalizeQuery(p.PlacesString) - if len(que) > 0 { - ret = append(ret, dbmodels.FTS5QueryRequest{ - Fields: []string{dbmodels.PLACES_TABLE, dbmodels.PLACE_STMT_FIELD}, - Query: que, - }) - } + req := dbmodels.IntoQueryRequests([]string{dbmodels.PLACES_TABLE, dbmodels.PLACE_STMT_FIELD}, que) + ret = append(ret, req...) } if p.RefsString != "" { que := dbmodels.NormalizeQuery(p.RefsString) - if len(que) > 0 { - ret = append(ret, dbmodels.FTS5QueryRequest{ - Fields: []string{dbmodels.REFERENCES_FIELD}, - Query: que, - }) - } + req := dbmodels.IntoQueryRequests([]string{dbmodels.REFERENCES_FIELD}, que) + ret = append(ret, req...) } if p.YearString != "" { que := dbmodels.NormalizeQuery(p.YearString) - if len(que) > 0 { - ret = append(ret, dbmodels.FTS5QueryRequest{ - Fields: []string{dbmodels.YEAR_FIELD}, - Query: dbmodels.NormalizeQuery(p.YearString), - }) - } + req := dbmodels.IntoQueryRequests([]string{dbmodels.YEAR_FIELD}, que) + ret = append(ret, req...) } return ret @@ -427,11 +400,13 @@ func (p SearchParameters) IsExtendedSearch() bool { return p.AnnotationsString != "" || p.PersonsString != "" || p.TitleString != "" || p.AlmString != "" || p.SeriesString != "" || p.PlacesString != "" || p.RefsString != "" || p.YearString != "" } -func (p SearchParameters) NormalizeQuery() []string { +func (p SearchParameters) NormalizeQuery() dbmodels.Query { return dbmodels.NormalizeQuery(p.Query) } type SearchResultBaende struct { + Queries []dbmodels.FTS5QueryRequest + // these are the sorted IDs for hits Hits []string Series map[string]*dbmodels.Series // <- Key: Series ID @@ -445,25 +420,57 @@ type SearchResultBaende struct { EntriesAgents map[string][]*dbmodels.REntriesAgents // <- Key: Entry ID } -func SimpleSearchBaende(app core.App, params SearchParameters) (*SearchResultBaende, error) { - fields := params.FieldSetBaende() - if len(fields) == 0 { - return nil, ErrNoQuery +func EmptyResultBaende() *SearchResultBaende { + return &SearchResultBaende{ + Hits: []string{}, + Series: make(map[string]*dbmodels.Series), + Entries: make(map[string]*dbmodels.Entry), + Places: make(map[string]*dbmodels.Place), + Agents: make(map[string]*dbmodels.Agent), + EntriesSeries: make(map[string][]*dbmodels.REntriesSeries), + SeriesEntries: make(map[string][]*dbmodels.REntriesSeries), + EntriesAgents: make(map[string][]*dbmodels.REntriesAgents), } +} - ids, err := dbmodels.FTS5Search(app, dbmodels.ENTRIES_TABLE, fields...) - if err != nil { - return nil, err +func SimpleSearchBaende(app core.App, params SearchParameters) (*SearchResultBaende, error) { + entries := []*dbmodels.Entry{} + queries := params.FieldSetBaende() + + if params.AlmString != "" { + e, err := dbmodels.Entries_MusenalmID(app, params.AlmString) + if err != nil && err == sql.ErrNoRows { + return EmptyResultBaende(), nil + } else if err != nil { + return nil, err + } + + entries = append(entries, e) + } else { + if len(queries) == 0 { + return nil, ErrNoQuery + } + + ids, err := dbmodels.FTS5Search(app, dbmodels.ENTRIES_TABLE, queries...) + if err != nil { + return nil, err + } + + resultids := []any{} + for _, id := range ids { + resultids = append(resultids, id.ID) + } + + e, err := dbmodels.Entries_IDs(app, resultids) + if err != nil { + return nil, err + } + entries = e } resultids := []any{} - for _, id := range ids { - resultids = append(resultids, id.ID) - } - - entries, err := dbmodels.Entries_IDs(app, resultids) - if err != nil { - return nil, err + for _, entry := range entries { + resultids = append(resultids, entry.Id) } entriesmap := make(map[string]*dbmodels.Entry) diff --git a/views/routes/suche/baende/body.gohtml b/views/routes/suche/baende/body.gohtml index 3f3ab63..38f6d66 100644 --- a/views/routes/suche/baende/body.gohtml +++ b/views/routes/suche/baende/body.gohtml @@ -7,7 +7,6 @@ Annotations bool Persons bool Title bool - Alm bool Series bool Places bool Refs bool @@ -52,7 +51,7 @@ {{ $isPersons := false }} {{ $isAnnotations := false }} -{{- $isAlm = or $model.parameters.Alm $model.parameters.AlmString -}} +{{- $isAlm = $model.parameters.AlmString -}} {{- $isTitle = or $model.parameters.Title $model.parameters.TitleString -}} {{- $isRefs = or $model.parameters.Refs $model.parameters.RefsString -}} {{- $isPlaces = or $model.parameters.Places $model.parameters.PlacesString -}} @@ -61,16 +60,33 @@ {{- $isPersons = or $model.parameters.Persons $model.parameters.PersonsString -}} {{- $isAnnotations = or $model.parameters.Annotations $model.parameters.AnnotationsString -}} -{{- $isBase := not (or $isAlm $isTitle $isRefs $isPlaces $isYear $isSeries $isPersons +{{- $isBase := not (or $isTitle $isRefs $isPlaces $isYear $isSeries $isPersons $isAnnotations) -}}