Added generic collections of objects. TODO: deserialization of metadata

HaWeb/Controllers/APIController.cs

@@ -116,7 +116,7 @@ public class APIController : Controller {
                     return UnprocessableEntity(ModelState);
 
                 //// 4. Stage: Is it a Hamann-Document? What kind?
-                var retdocs = _xmlService.ProbeHamannFile(xdocument, ModelState);
+                var retdocs = _xmlService.ProbeFile(xdocument, ModelState);
                 if (!ModelState.IsValid || retdocs == null || !retdocs.Any())
                     return UnprocessableEntity(ModelState);
 

HaWeb/Controllers/SucheController.cs

@@ -6,17 +6,20 @@ using HaDocument.Interfaces;
 using HaDocument.Models;
 using HaXMLReader.Interfaces;
 using System.Collections.Specialized;
+using HaWeb.XMLParser;
 
 namespace HaWeb.Controllers;
 
 public class SucheController : Controller {
     private IHaDocumentWrappper _lib;
     private IReaderService _readerService;
+    private IXMLService _xmlService;
     private int _lettersForPage;
 
-    public SucheController(IHaDocumentWrappper lib, IReaderService readerService, IConfiguration config) {
+    public SucheController(IHaDocumentWrappper lib, IReaderService readerService, IXMLService service, IConfiguration config) {
         _lib = lib;
         _readerService = readerService;
+        _xmlService = service;
         _lettersForPage = config.GetValue<int>("LettersOnPage");
     }
 
@@ -33,7 +36,6 @@ public class SucheController : Controller {
 
     [Route("Suche/{zhvolume}/{zhpage}")]
     public IActionResult GoToZH(string zhvolume, string zhpage) {
-        // TODO: Bug in letter parsing: dictionary is WRONG!
         if (String.IsNullOrWhiteSpace(zhvolume) || String.IsNullOrWhiteSpace(zhpage)) return _error404();
         zhvolume = zhvolume.Trim();
         zhpage = zhpage.Trim();
@@ -68,7 +70,7 @@ public class SucheController : Controller {
         List<IGrouping<int, Meta>>? metasbyyear = null;
         if (search != null) {
             search = search.Trim();
-            var res = _lib.SearchLetters(search, _readerService);
+            var res = _xmlService.SearchCollection("letters", search, _readerService);
             if (res == null || !res.Any()) return _error404();
             var ret = res.ToDictionary(
                 x => x.Index, 

HaWeb/FileHelpers/HaDocumentWrapper.cs

@@ -6,19 +6,22 @@ using System.Collections.Concurrent;
 using System.Threading.Tasks;
 using HaXMLReader.Interfaces;
 using HaWeb.SearchHelpers;
+using HaWeb.XMLParser;
 using System.Text;
 
 public class HaDocumentWrapper : IHaDocumentWrappper {
     private ILibrary Library;
     private IXMLProvider _xmlProvider;
+    private IXMLService _xmlService;
 
     public int StartYear { get; private set; }
     public int EndYear { get; private set; }
 
-    public List<SearchHelpers.SeachableItem>? SearchableLetters { get; private set; }
+    // public List<SearchHelpers.CollectedItem>? SearchableLetters { get; private set; }
 
-    public HaDocumentWrapper(IXMLProvider xmlProvider, IConfiguration configuration) {
+    public HaDocumentWrapper(IXMLProvider xmlProvider, IXMLService service, IConfiguration configuration) {
         _xmlProvider = xmlProvider;
+        _xmlService = service;
         StartYear = configuration.GetValue<int>("AvailableStartYear");
         EndYear = configuration.GetValue<int>("AvailableEndYear");
         var filelist = xmlProvider.GetHamannFiles();
@@ -41,52 +44,26 @@ public class HaDocumentWrapper : IHaDocumentWrappper {
     }
 
     public ILibrary? SetLibrary(string filepath, ModelStateDictionary? ModelState = null) {
+        var sw = new System.Diagnostics.Stopwatch();
+        sw.Start();
         try {
             Library = HaDocument.Document.Create(new HaWeb.Settings.HaDocumentOptions() { HamannXMLFilePath = filepath, AvailableYearRange = (StartYear, EndYear) });
         } catch (Exception ex) {
             if (ModelState != null) ModelState.AddModelError("Error", "Das Dokument konnte nicht geparst werden: " + ex.Message);
             return null;
         }
+        sw.Stop();
+        Console.WriteLine("ILIB: " + sw.ElapsedMilliseconds);
+        sw.Restart();
 
-        var searchableletters = new ConcurrentBag<SearchHelpers.SeachableItem>();
+        if (_xmlService != null) 
-        var letters = Library.Letters.Values;
+            _xmlService.SetInProduction(System.Xml.Linq.XDocument.Load(filepath, System.Xml.Linq.LoadOptions.PreserveWhitespace));
-
+        sw.Stop();
-        Parallel.ForEach(letters, letter => {
+        Console.WriteLine("COLLECTIONS: " + sw.ElapsedMilliseconds);
-            var o = new SearchHelpers.SeachableItem(letter.Index, _prepareSearch(letter));
-            searchableletters.Add(o);
-        });
-
-        this.SearchableLetters = searchableletters.ToList();
 
         return Library;
     }
 
-    public List<(string Index, List<(string Page, string Line, string Preview)> Results)>? SearchLetters(string searchword, IReaderService reader) {
-        if (SearchableLetters == null) return null;
-        var res = new ConcurrentBag<(string Index, List<(string Page, string Line, string preview)> Results)>();
-        var sw = StringHelpers.NormalizeWhiteSpace(searchword.Trim());
-        Parallel.ForEach(SearchableLetters, (letter) => {
-            var state = new SearchState(sw);
-            var rd = reader.RequestStringReader(letter.SearchText);
-            var parser = new HaWeb.HTMLParser.LineXMLHelper<SearchState>(state, rd, new StringBuilder(), null, null, null, SearchRules.TextRules, SearchRules.WhitespaceRules);
-            rd.Read();
-            if (state.Results != null)
-                res.Add((
-                    letter.Index,
-                    state.Results.Select(x => (
-                        x.Page,
-                        x.Line,
-                        parser.Lines != null ?
-                            parser.Lines
-                            .Where(y => y.Page == x.Page && y.Line == x.Line)
-                            .Select(x => x.Text)
-                            .FirstOrDefault(string.Empty)
-                            : ""
-                    )).ToList()));
-        });
-        return res.ToList();
-    }
-
     public ILibrary GetLibrary() {
         return Library;
     }

HaWeb/FileHelpers/IHaDocumentWrapper.cs

@@ -7,5 +7,4 @@ public interface IHaDocumentWrappper {
     public ILibrary ResetLibrary();
     public ILibrary? SetLibrary(string filepath, ModelStateDictionary ModelState);
     public ILibrary GetLibrary();
-    public List<(string Index, List<(string Page, string Line, string Preview)> Results)>? SearchLetters(string searchword, IReaderService reader);
 }

HaWeb/Models/CollectedItem.cs

@@ -0,0 +1,20 @@
+namespace HaWeb.Models;
+using HaWeb.SearchHelpers;
+using HaWeb.XMLParser;
+using System.Xml.Linq;
+
+public class CollectedItem : ISearchable {
+    public string Index { get; private set; }
+    public string Collection { get; private set; }
+    public string? SearchText { get; private set; }
+    public XElement ELement { get; private set; }
+    public IXMLRoot Root { get; private set; }
+
+    public CollectedItem(string index, XElement element, IXMLRoot root, string collection, string? searchtext = null) {
+        this.Index = index;
+        this.SearchText = searchtext;
+        this.Collection = collection;
+        this.Root = root;
+        this.ELement = element;
+    }
+}

HaWeb/Models/FileList.cs

@@ -32,9 +32,10 @@ public class FileList {
 
     public FileList Clone() {
         var ret = new FileList(this.XMLRoot);
-        foreach (var file in _Files) {
+        if (_Files != null)
-            ret.Add(file);
+            foreach (var file in _Files) {
-        }
+                ret.Add(file);
+            }
         return ret;
     }
 }

HaWeb/SearchHelpers/SearchableItem.cs

@@ -1,11 +0,0 @@
-namespace HaWeb.SearchHelpers;
-
-public class SeachableItem : ISearchable {
-    public string Index { get; private set; }
-    public string SearchText { get; private set; }
-
-    public SeachableItem(string index, string searchtext) {
-        this.Index = index;
-        this.SearchText = searchtext;
-    }
-}

HaWeb/Settings/XMLRoots/CommentRoot.cs

@@ -7,13 +7,23 @@ public class CommentRoot : HaWeb.XMLParser.IXMLRoot {
     public string Type { get; } = "Register";
     public string Prefix { get; } = "register";
     public string[] XPathContainer { get; } = { ".//data//kommentare/kommcat", ".//kommentare/kommcat" };
+    public (string Key, string xPath, Func<XElement, string?> KeyFunc, bool Searchable)[]? XPathCollection { get; } = { 
+        ("comments-register", "/opus/data/kommentare/kommcat[@value='neuzeit']/kommentar", GetKey, true),
+        ("comments-register", "/opus/kommentare/kommcat[@value='neuzeit']/kommentar", GetKey, true),
+        ("comments-edition", "/opus/data/kommentare/kommcat[@value='editionen']/kommentar", GetKey, true),
+        ("comments-edition", "/opus/kommentare/kommcat[@value='editionen']/kommentar", GetKey, true),
+        ("comments-forschung", "/opus/data/kommentare/kommcat[@value='forschung']/kommentar", GetKey, true),
+        ("comments-forschung", "/opus/kommentare/kommcat[@value='forschung']/kommentar", GetKey, true),
+        ("comments-bibel", "/opus/data/kommentare/kommcat[@value='bibel']/kommentar", GetKey, false),
+        ("comments-bibel", "/opus/kommentare/kommcat[@value='bibel']/kommentar", GetKey, false),
+    };
 
     public Predicate<XElement> IsCollectedObject { get; } = (elem) => {
         if (elem.Name == "kommentar") return true;
         else return false;
     };
 
-    public Func<XElement, string?> GetKey { get; } = (elem) => {
+    public static Func<XElement, string?> GetKey { get; } = (elem) => {
         var index = elem.Attribute("id");
         if (index != null && !String.IsNullOrWhiteSpace(index.Value))
             return index.Value;

HaWeb/Settings/XMLRoots/DescriptionsRoot.cs

@@ -7,18 +7,22 @@ public class DescriptionsRoot : HaWeb.XMLParser.IXMLRoot {
     public string Type { get; } = "Metadaten";
     public string Prefix { get; } = "metadaten";
     public string[] XPathContainer { get; } = { ".//data/descriptions", ".//descriptions" };
-
+    public (string Key, string xPath, Func<XElement, string?> KeyFunc, bool Searchable)[]? XPathCollection { get; } = {
+        ("metas", "/opus/descriptions/letterDesc", GetKey, false),
+        ("metas", "/opus/data/descriptions/letterDesc", GetKey, false)
+    };
+    
     public Predicate<XElement> IsCollectedObject { get; } = (elem) => {
         if (elem.Name == "letterDesc") return true;
         return false;
     };
 
-    // public Func<XElement, string?> GetKey { get; } = (elem) => {
+    public static Func<XElement, string?> GetKey { get; } = (elem) => {
-    //     var index = elem.Attribute("ref");
+        var index = elem.Attribute("ref");
-    //     if (index != null && !String.IsNullOrWhiteSpace(index.Value))
+        if (index != null && !String.IsNullOrWhiteSpace(index.Value))
-    //         return index.Value;
+            return index.Value;
-    //     else return null;
+        return null;
-    // };
+    };
 
     public List<(string, string?)>? GenerateFields(XMLRootDocument document) {
         return null;

HaWeb/Settings/XMLRoots/DocumentRoot.cs

@@ -8,13 +8,17 @@ public class DocumentRoot : HaWeb.XMLParser.IXMLRoot {
     public string Type { get; } = "Brieftext";
     public string Prefix { get; } = "brieftext";
     public string[] XPathContainer { get; } = { ".//data/document", ".//document" };
+    public (string Key, string xPath, Func<XElement, string?> KeyFunc, bool Searchable)[]? XPathCollection { get; } = { 
+        ("letters", "/opus/data/document/letterText", GetKey, true),
+        ("letters", "/opus/document/letterText", GetKey, true)
+    };
 
     public Predicate<XElement> IsCollectedObject { get; } = (elem) => {
         if (elem.Name == "letterText") return true;
         else return false;
     };
 
-    public Func<XElement, string?> GetKey { get; } = (elem) => {
+    public static Func<XElement, string?> GetKey { get; } = (elem) => {
         var index = elem.Attribute("index");
         if (index != null && !String.IsNullOrWhiteSpace(index.Value))
             return index.Value;

HaWeb/Settings/XMLRoots/EditsRoot.cs

@@ -7,13 +7,17 @@ public class EditsRoot : HaWeb.XMLParser.IXMLRoot {
     public string Type { get; } = "Texteingriffe";
     public string Prefix { get; } = "texteingriffe";
     public string[] XPathContainer { get; } = { ".//data/edits", ".//edits" };
-
+    public (string Key, string xPath, Func<XElement, string?> KeyFunc, bool Searchable)[]? XPathCollection { get; } = { 
+        ("edits", "/data/edits/editreason", GetKey, true),
+        ("edits", "/edits/editreason", GetKey, true)
+    };
+    
     public Predicate<XElement> IsCollectedObject { get; } = (elem) => {
         if (elem.Name == "editreason") return true;
         else return false;
     };
 
-    public Func<XElement, string?> GetKey { get; } = (elem) => {
+    public static Func<XElement, string?> GetKey { get; } = (elem) => {
         var index = elem.Attribute("index");
         if (index != null && !String.IsNullOrWhiteSpace(index.Value))
             return index.Value;

HaWeb/Settings/XMLRoots/MarginalsRoot.cs

@@ -7,13 +7,17 @@ public class MarginalsRoot : HaWeb.XMLParser.IXMLRoot {
     public string Type { get; } = "Stellenkommentar";
     public string Prefix { get; } = "stellenkommentar";
     public string[] XPathContainer { get; } = { ".//data/marginalien", ".//marginalien" };
+    public (string Key, string xPath, Func<XElement, string?> KeyFunc, bool Searchable)[]? XPathCollection { get; } = { 
+        ("marginals", "/data/marginalien/marginal", GetKey, true),
+        ("marginals", "/marginalien/marginal", GetKey, true)
+    };
 
     public Predicate<XElement> IsCollectedObject { get; } = (elem) => {
         if (elem.Name == "marginal") return true;
         else return false;
     };
 
-    public Func<XElement, string?> GetKey { get; } = (elem) => {
+    public static Func<XElement, string?> GetKey { get; } = (elem) => {
         var index = elem.Attribute("index");
         if (index != null && !String.IsNullOrWhiteSpace(index.Value))
             return index.Value;

HaWeb/Settings/XMLRoots/ReferencesRoot.cs

@@ -7,6 +7,14 @@ public class ReferencesRoot : HaWeb.XMLParser.IXMLRoot {
     public string Type { get; } = "Personen / Orte";
     public string Prefix { get; } = "personenorte";
     public string[] XPathContainer { get; } = { ".//data/definitions", ".//definitions" };
+    public (string Key, string xPath, Func<XElement, string?> KeyFunc, bool Searchable)[]? XPathCollection { get; } = {
+        ("person-definitions", "/opus/data/definitions/personDefs/personDef", GetKey, false),
+        ("person-definitions", "/opus/definitions/personDefs/personDef", GetKey, false),
+        ("hand-definitions", "/opus/data/definitions/handDefs/handDef", GetKey, false),
+        ("hand-definitions", "/opus/definitions/handDefs/handDef", GetKey, false),
+        ("location-definitions", "/opus/data/definitions/locationDefs/locationDef", GetKey, false),
+        ("location-definitions", "/opus/definitions/locationDefs/locationDef", GetKey, false)
+    };
 
     public Predicate<XElement> IsCollectedObject { get; } = (elem) => {
         if (elem.Name == "personDefs" || elem.Name == "structureDefs" || elem.Name == "handDefs" || elem.Name == "locationDefs")
@@ -14,8 +22,8 @@ public class ReferencesRoot : HaWeb.XMLParser.IXMLRoot {
         return false;
     };
 
-    public Func<XElement, string?> GetKey { get; } = (elem) => {
+    public static Func<XElement, string?> GetKey { get; } = (elem) => {
-        return elem.Name.ToString();
+        return elem.Attribute("index") != null ? elem.Attribute("index")!.Value : null;
     };
 
     public List<(string, string?)>? GenerateFields(XMLRootDocument document) {

HaWeb/Settings/XMLRoots/TraditionsRoot.cs

@@ -7,17 +7,21 @@ public class TraditionsRoot : HaWeb.XMLParser.IXMLRoot {
     public string Type { get; } = "Überlieferung";
     public string Prefix { get; } = "ueberlieferung";
     public string[] XPathContainer { get; } = { ".//data/traditions", ".//traditions" };
+    public (string Key, string xPath, Func<XElement, string?> KeyFunc, bool Searchable)[]? XPathCollection { get; } = { 
+        ("tradition", "/opus/data/traditions/letterTradition", GetKey, true),
+        ("tradition", "/opus/traditions/letterTradition", GetKey, true)
+    };
 
     public Predicate<XElement> IsCollectedObject { get; } = (elem) => {
         if (elem.Name == "letterTradition") return true;
         else return false;
     };
 
-    public Func<XElement, string?> GetKey { get; } = (elem) => {
+    public static Func<XElement, string?> GetKey { get; } = (elem) => {
         var index = elem.Attribute("ref");
         if (index != null && !String.IsNullOrWhiteSpace(index.Value))
             return index.Value;
-        else return null;
+        return null;
     };
 
     public List<(string, string?)>? GenerateFields(XMLRootDocument document) {

HaWeb/XMLParser/IXMLRoot.cs

@@ -13,7 +13,15 @@ public interface IXMLRoot {
     // XPaths to determine if container is present
     public abstract string[] XPathContainer { get; }
 
-    // Tag Name of child objects to be collected 
+    // Collections of Elements to be created from this Root
+    // Key: the key under which the element(s) will be files
+    // xPath: the (absolute) XPath to the element(s)
+    // KeyFunc: How to extrect an identifier for the single element in the collection
+    // Searchable: Will the element be indexed for full-text-search?
+    public abstract (string Key, string xPath, Func<XElement, string?> KeyFunc, bool Searchable)[]? XPathCollection { get; }
+
+    // Determines child objects to be collected 
+    // (deprecated see collections above; only used internally)
     public abstract Predicate<XElement> IsCollectedObject { get; }
 
     // Gets the Key of a collected object

HaWeb/XMLParser/IXMLService.cs

@@ -2,12 +2,13 @@ namespace HaWeb.XMLParser;
 using System.Xml.Linq;
 using Microsoft.AspNetCore.Mvc.ModelBinding;
 using HaWeb.Models;
+using HaXMLReader.Interfaces;
 
 public interface IXMLService {
     public IXMLRoot? GetRoot(string name);
     public List<IXMLRoot>? GetRootsList();
     public Dictionary<string, IXMLRoot>? GetRootsDictionary();
-    public List<XMLRootDocument>? ProbeHamannFile(XDocument document, ModelStateDictionary ModelState);
+    public List<XMLRootDocument>? ProbeFile(XDocument document, ModelStateDictionary ModelState);
     public Dictionary<string, FileList?>? GetUsedDictionary();
     public XElement? MergeUsedDocuments(ModelStateDictionary ModelState);
     public void Use(XMLRootDocument doc);
@@ -17,4 +18,6 @@ public interface IXMLService {
     public void UnUse(string prefix);
     public void UnUseProduction();
     public void SetInProduction();
+    public void SetInProduction(XDocument document);
+    public List<(string Index, List<(string Page, string Line, string Preview)> Results)>? SearchCollection(string collection, string searchword, IReaderService reader);
 }

HaWeb/XMLParser/XMLService.cs

@@ -1,7 +1,13 @@
 namespace HaWeb.XMLParser;
 using System.Xml.Linq;
+using System.Xml.XPath;
 using Microsoft.AspNetCore.Mvc.ModelBinding;
 using HaWeb.Models;
+using HaWeb.SearchHelpers;
+using System.Collections.Concurrent;
+using System.Threading.Tasks;
+using System.Text;
+using HaXMLReader.Interfaces;
 
 public class XMLService : IXMLService {
     private Dictionary<string, FileList?>? _Used;
@@ -9,6 +15,9 @@ public class XMLService : IXMLService {
 
     private Stack<Dictionary<string, FileList?>>? _InProduction;
 
+    private Dictionary<string, Dictionary<string, CollectedItem>> _collectedProduction;
+    private Dictionary<string, Dictionary<string, CollectedItem>> _collectedUsed;
+
     public XMLService() {
         // Getting all classes which implement IXMLRoot for possible document endpoints
         var types = _GetAllTypesThatImplementInterface<IXMLRoot>().ToList();
@@ -50,9 +59,63 @@ public class XMLService : IXMLService {
         _InProduction.Push(inProduction);
     }
 
+    public void SetInProduction(XDocument document) {
+        if (document == null || _Roots == null) return;
+        var ret = new ConcurrentDictionary<string, ConcurrentDictionary<string, CollectedItem>>();
+        Parallel.ForEach(_Roots, (root) => {
+            if (root.Value.XPathCollection != null)
+                foreach (var coll in root.Value.XPathCollection) {
+                    var elem = document.XPathSelectElements(coll.xPath);
+                    if (elem != null && elem.Any()) {
+                        if (!ret.ContainsKey(coll.Key))
+                            ret[coll.Key] = new ConcurrentDictionary<string, CollectedItem>();
+                        foreach(var e in elem) {
+                            var k = coll.KeyFunc(e);
+                            if (k != null) {
+                                var searchtext = coll.Searchable ? 
+                                    StringHelpers.NormalizeWhiteSpace(e.ToString(), ' ', false) : 
+                                    null;
+                                ret[coll.Key][k] = new CollectedItem(k, e, root.Value, coll.Key, searchtext);
+                            }
+                        }
+                    }
+                }
+        });       
+        _collectedProduction = ret.ToDictionary(x => x.Key, y => y.Value.ToDictionary(z => z.Key, f => f.Value, null), null);
+    }
+
+     public List<(string Index, List<(string Page, string Line, string Preview)> Results)>? SearchCollection(string collection, string searchword, IReaderService reader) {
+        if (!_collectedProduction.ContainsKey(collection)) return null;
+        var searchableObjects = _collectedProduction[collection];
+        var res = new ConcurrentBag<(string Index, List<(string Page, string Line, string preview)> Results)>();
+        var sw = StringHelpers.NormalizeWhiteSpace(searchword.Trim());
+        Parallel.ForEach(searchableObjects, (obj) => {
+            if (obj.Value.SearchText != null) {
+                var state = new SearchState(sw);
+                var rd = reader.RequestStringReader(obj.Value.SearchText);
+                var parser = new HaWeb.HTMLParser.LineXMLHelper<SearchState>(state, rd, new StringBuilder(), null, null, null, SearchRules.TextRules, SearchRules.WhitespaceRules);
+                rd.Read();
+                if (state.Results != null)
+                    res.Add((
+                        obj.Value.Index,
+                        state.Results.Select(x => (
+                            x.Page,
+                            x.Line,
+                            parser.Lines != null ?
+                                parser.Lines
+                                .Where(y => y.Page == x.Page && y.Line == x.Line)
+                                .Select(x => x.Text)
+                                .FirstOrDefault(string.Empty)
+                                : ""
+                        )).ToList()));
+            }
+        });
+        return res.ToList();
+    }
+
     public void UnUseProduction() => this._InProduction = null;
 
-    public List<XMLRootDocument>? ProbeHamannFile(XDocument document, ModelStateDictionary ModelState) {
+    public List<XMLRootDocument>? ProbeFile(XDocument document, ModelStateDictionary ModelState) {
         if (document.Root!.Name != "opus") {
             ModelState.AddModelError("Error", "A valid Hamann-Docuemnt must begin with <opus>");
             return null;