From 83b4e5327ad9591670fd86dba42805bb52110891 Mon Sep 17 00:00:00 2001
From: schnulller <schnulller@mailbox.org>
Date: Wed, 22 Jun 2022 14:57:09 +0200
Subject: [PATCH] Added generic collections of objects

---
 HaWeb/Controllers/SucheController.cs | 13 ++++++++-----
 HaWeb/XMLParser/IXMLRoot.cs          |  9 ++++++++-
 HaWeb/XMLParser/XMLService.cs        | 21 +++++++++++++++------
 3 files changed, 31 insertions(+), 12 deletions(-)

diff --git a/HaWeb/Controllers/SucheController.cs b/HaWeb/Controllers/SucheController.cs
index 31e0a66..694639f 100644
--- a/HaWeb/Controllers/SucheController.cs
+++ b/HaWeb/Controllers/SucheController.cs
@@ -69,19 +69,22 @@ public class SucheController : Controller {
         var lib = _lib.GetLibrary();
         List<IGrouping<int, Meta>>? metasbyyear = null;
         if (search != null) {
+            var stopwatch = new System.Diagnostics.Stopwatch();
+            stopwatch.Start();
             search = search.Trim();
             var res = _xmlService.SearchCollection("letters", search, _readerService);
             if (res == null || !res.Any()) return _error404();
             var ret = res.ToDictionary(
-                x => x.Index, 
+                x => x.Index,
                 x => x.Results
-                    .Select(y => new SearchResult(search, x.Index) { Page = y.Page, Line = y.Line, Preview = y.Preview})
+                    .Select(y => new SearchResult(search, x.Index) { Page = y.Page, Line = y.Line, Preview = y.Preview })
                     .ToList()
                 );
             var keys = res.Select(x => x.Index).Where(x => lib.Metas.ContainsKey(x)).Select(x => lib.Metas[x]);
             var letters = keys.ToLookup(x => x.Sort.Year).OrderBy(x => x.Key).ToList();
+            stopwatch.Stop();
+            Console.WriteLine("SEARCH: " + stopwatch.ElapsedMilliseconds);
             return _paginateSend(lib, page, letters, null, null, null, search, ret);
-
         }
         metasbyyear = lib.MetasByYear.OrderBy(x => x.Key).ToList();
         return _paginateSend(lib, page, metasbyyear);
@@ -105,7 +108,7 @@ public class SucheController : Controller {
     private List<(string Key, string Person)> _getAvailablePersons(ILibrary lib) {
         return lib.Persons
             .OrderBy(x => x.Value.Surname)
-            .ThenBy(x =>  x.Value.Prename)
+            .ThenBy(x => x.Value.Prename)
             .Select(x => (x.Key, x.Value.Name))
             .ToList();
     }
@@ -121,7 +124,7 @@ public class SucheController : Controller {
             ParsedReceivers = HTMLHelpers.StringHelpers.GetEnumerationString(recivers)
         };
     }
-    
+
     private List<(int StartYear, int EndYear)>? _paginate(List<IGrouping<int, Meta>>? letters) {
         if (letters == null || !letters.Any()) return null;
         List<(int StartYear, int EndYear)>? res = null;
diff --git a/HaWeb/XMLParser/IXMLRoot.cs b/HaWeb/XMLParser/IXMLRoot.cs
index 7a0b9b3..7f17f56 100644
--- a/HaWeb/XMLParser/IXMLRoot.cs
+++ b/HaWeb/XMLParser/IXMLRoot.cs
@@ -17,8 +17,15 @@ public interface IXMLRoot {
     // Key: the key under which the element(s) will be files
     // xPath: the (absolute) XPath to the element(s)
     // KeyFunc: How to extrect an identifier for the single element in the collection
+    // LookupsFunc: Function to generate metadata fields for the object, which will then in turn be a possibility to seach and filter without parsing
     // Searchable: Will the element be indexed for full-text-search?
-    public abstract (string Key, string xPath, Func<XElement, string?> KeyFunc, bool Searchable)[]? XPathCollection { get; }
+    public abstract (
+        string Key, 
+        string xPath,
+        Func<XElement, string?> KeyFunc, 
+        // Func<XElement, Dictionary<string, string[]>> LookupsFunc,
+        bool Searchable
+    )[]? XPathCollection { get; }
 
     // Determines child objects to be collected 
     // (deprecated see collections above; only used internally)
diff --git a/HaWeb/XMLParser/XMLService.cs b/HaWeb/XMLParser/XMLService.cs
index 7bec00f..74435cf 100644
--- a/HaWeb/XMLParser/XMLService.cs
+++ b/HaWeb/XMLParser/XMLService.cs
@@ -61,15 +61,22 @@ public class XMLService : IXMLService {
 
     public void SetInProduction(XDocument document) {
         if (document == null || _Roots == null) return;
-        var ret = new ConcurrentDictionary<string, ConcurrentDictionary<string, CollectedItem>>();
-        Parallel.ForEach(_Roots, (root) => {
+        int numProcs = Environment.ProcessorCount;
+        int concurrencyLevel = numProcs * 2;
+        int startingSize = 2909;
+        int startingSizeAllCollections = 23;
+        var ret = new ConcurrentDictionary<string, ConcurrentDictionary<string, CollectedItem>>(concurrencyLevel, startingSizeAllCollections);
+        // Note Parallelization brings almost nothing to the table (on a laptop) here and below.
+        // Parallel.ForEach(_Roots, (root) => {
+        foreach (var root in _Roots) {
             if (root.Value.XPathCollection != null)
                 foreach (var coll in root.Value.XPathCollection) {
                     var elem = document.XPathSelectElements(coll.xPath);
                     if (elem != null && elem.Any()) {
                         if (!ret.ContainsKey(coll.Key))
-                            ret[coll.Key] = new ConcurrentDictionary<string, CollectedItem>();
-                        foreach(var e in elem) {
+                            ret[coll.Key] = new ConcurrentDictionary<string, CollectedItem>(concurrencyLevel, startingSize);
+                        Parallel.ForEach(elem, (e) => {
+                        // foreach(var e in elem) {
                             var k = coll.KeyFunc(e);
                             if (k != null) {
                                 var searchtext = coll.Searchable ? 
@@ -77,10 +84,12 @@ public class XMLService : IXMLService {
                                     null;
                                 ret[coll.Key][k] = new CollectedItem(k, e, root.Value, coll.Key, searchtext);
                             }
-                        }
+                        // }
+                        });
                     }
                 }
-        });       
+        }
+        // });       
         _collectedProduction = ret.ToDictionary(x => x.Key, y => y.Value.ToDictionary(z => z.Key, f => f.Value, null), null);
     }