Theodor-Springmann-Stiftung/hamann-ausgabe-core
1
0
Fork 0
mirror of https://github.com/Theodor-Springmann-Stiftung/hamann-ausgabe-core.git synced 2025-10-29 09:15:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added generic collections of objects. TODO: deserialization of metadata

Browse Source
This commit is contained in:
schnulller
2022-06-19 23:20:35 +02:00
parent d8a4fe7790
commit ba3d63ffba
17 changed files with 175 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
HaWeb/Controllers/APIController.cs
View File

@@ -116,7 +116,7 @@ public class APIController : Controller {
return UnprocessableEntity(ModelState);
//// 4. Stage: Is it a Hamann-Document? What kind?
var retdocs = _xmlService.ProbeHamannFile(xdocument, ModelState);
var retdocs = _xmlService.ProbeFile(xdocument, ModelState);
if (!ModelState.IsValid || retdocs == null || !retdocs.Any())
return UnprocessableEntity(ModelState);

8
HaWeb/Controllers/SucheController.cs
View File

@@ -6,17 +6,20 @@ using HaDocument.Interfaces;
using HaDocument.Models;
using HaXMLReader.Interfaces;
using System.Collections.Specialized;
using HaWeb.XMLParser;
namespace HaWeb.Controllers;
public class SucheController : Controller {
private IHaDocumentWrappper _lib;
private IReaderService _readerService;
private IXMLService _xmlService;
private int _lettersForPage;
public SucheController(IHaDocumentWrappper lib, IReaderService readerService, IConfiguration config) {
public SucheController(IHaDocumentWrappper lib, IReaderService readerService, IXMLService service, IConfiguration config) {
_lib = lib;
_readerService = readerService;
_xmlService = service;
_lettersForPage = config.GetValue<int>("LettersOnPage");
}
@@ -33,7 +36,6 @@ public class SucheController : Controller {
[Route("Suche/{zhvolume}/{zhpage}")]
public IActionResult GoToZH(string zhvolume, string zhpage) {
// TODO: Bug in letter parsing: dictionary is WRONG!
if (String.IsNullOrWhiteSpace(zhvolume) || String.IsNullOrWhiteSpace(zhpage)) return _error404();
zhvolume = zhvolume.Trim();
zhpage = zhpage.Trim();
@@ -68,7 +70,7 @@ public class SucheController : Controller {
List<IGrouping<int, Meta>>? metasbyyear = null;
if (search != null) {
search = search.Trim();
var res = _lib.SearchLetters(search, _readerService);
var res = _xmlService.SearchCollection("letters", search, _readerService);
if (res == null || !res.Any()) return _error404();
var ret = res.ToDictionary(
x => x.Index,

51
HaWeb/FileHelpers/HaDocumentWrapper.cs
View File

@@ -6,19 +6,22 @@ using System.Collections.Concurrent;
using System.Threading.Tasks;
using HaXMLReader.Interfaces;
using HaWeb.SearchHelpers;
using HaWeb.XMLParser;
using System.Text;
public class HaDocumentWrapper : IHaDocumentWrappper {
private ILibrary Library;
private IXMLProvider _xmlProvider;
private IXMLService _xmlService;
public int StartYear { get; private set; }
public int EndYear { get; private set; }
public List<SearchHelpers.SeachableItem>? SearchableLetters { get; private set; }
// public List<SearchHelpers.CollectedItem>? SearchableLetters { get; private set; }
public HaDocumentWrapper(IXMLProvider xmlProvider, IConfiguration configuration) {
public HaDocumentWrapper(IXMLProvider xmlProvider, IXMLService service, IConfiguration configuration) {
_xmlProvider = xmlProvider;
_xmlService = service;
StartYear = configuration.GetValue<int>("AvailableStartYear");
EndYear = configuration.GetValue<int>("AvailableEndYear");
var filelist = xmlProvider.GetHamannFiles();
@@ -41,52 +44,26 @@ public class HaDocumentWrapper : IHaDocumentWrappper {
}
public ILibrary? SetLibrary(string filepath, ModelStateDictionary? ModelState = null) {
var sw = new System.Diagnostics.Stopwatch();
sw.Start();
try {
Library = HaDocument.Document.Create(new HaWeb.Settings.HaDocumentOptions() { HamannXMLFilePath = filepath, AvailableYearRange = (StartYear, EndYear) });
} catch (Exception ex) {
if (ModelState != null) ModelState.AddModelError("Error", "Das Dokument konnte nicht geparst werden: " + ex.Message);
return null;
}
sw.Stop();
Console.WriteLine("ILIB: " + sw.ElapsedMilliseconds);
sw.Restart();
var searchableletters = new ConcurrentBag<SearchHelpers.SeachableItem>();
var letters = Library.Letters.Values;
Parallel.ForEach(letters, letter => {
var o = new SearchHelpers.SeachableItem(letter.Index, _prepareSearch(letter));
searchableletters.Add(o);
});
this.SearchableLetters = searchableletters.ToList();
if (_xmlService != null)
_xmlService.SetInProduction(System.Xml.Linq.XDocument.Load(filepath, System.Xml.Linq.LoadOptions.PreserveWhitespace));
sw.Stop();
Console.WriteLine("COLLECTIONS: " + sw.ElapsedMilliseconds);
return Library;
}
public List<(string Index, List<(string Page, string Line, string Preview)> Results)>? SearchLetters(string searchword, IReaderService reader) {
if (SearchableLetters == null) return null;
var res = new ConcurrentBag<(string Index, List<(string Page, string Line, string preview)> Results)>();
var sw = StringHelpers.NormalizeWhiteSpace(searchword.Trim());
Parallel.ForEach(SearchableLetters, (letter) => {
var state = new SearchState(sw);
var rd = reader.RequestStringReader(letter.SearchText);
var parser = new HaWeb.HTMLParser.LineXMLHelper<SearchState>(state, rd, new StringBuilder(), null, null, null, SearchRules.TextRules, SearchRules.WhitespaceRules);
rd.Read();
if (state.Results != null)
res.Add((
letter.Index,
state.Results.Select(x => (
x.Page,
x.Line,
parser.Lines != null ?
parser.Lines
.Where(y => y.Page == x.Page && y.Line == x.Line)
.Select(x => x.Text)
.FirstOrDefault(string.Empty)
: ""
)).ToList()));
});
return res.ToList();
}
public ILibrary GetLibrary() {
return Library;
}

1
HaWeb/FileHelpers/IHaDocumentWrapper.cs
View File

@@ -7,5 +7,4 @@ public interface IHaDocumentWrappper {
public ILibrary ResetLibrary();
public ILibrary? SetLibrary(string filepath, ModelStateDictionary ModelState);
public ILibrary GetLibrary();
public List<(string Index, List<(string Page, string Line, string Preview)> Results)>? SearchLetters(string searchword, IReaderService reader);
}

20
HaWeb/Models/CollectedItem.cs Normal file
View File

@@ -0,0 +1,20 @@
namespace HaWeb.Models;
using HaWeb.SearchHelpers;
using HaWeb.XMLParser;
using System.Xml.Linq;
public class CollectedItem : ISearchable {
public string Index { get; private set; }
public string Collection { get; private set; }
public string? SearchText { get; private set; }
public XElement ELement { get; private set; }
public IXMLRoot Root { get; private set; }
public CollectedItem(string index, XElement element, IXMLRoot root, string collection, string? searchtext = null) {
this.Index = index;
this.SearchText = searchtext;
this.Collection = collection;
this.Root = root;
this.ELement = element;
}
}

1
HaWeb/Models/FileList.cs
View File

@@ -32,6 +32,7 @@ public class FileList {
public FileList Clone() {
var ret = new FileList(this.XMLRoot);
if (_Files != null)
foreach (var file in _Files) {
ret.Add(file);
}

11
HaWeb/SearchHelpers/SearchableItem.cs
View File

@@ -1,11 +0,0 @@
namespace HaWeb.SearchHelpers;
public class SeachableItem : ISearchable {
public string Index { get; private set; }
public string SearchText { get; private set; }
public SeachableItem(string index, string searchtext) {
this.Index = index;
this.SearchText = searchtext;
}
}

12
HaWeb/Settings/XMLRoots/CommentRoot.cs
View File

@@ -7,13 +7,23 @@ public class CommentRoot : HaWeb.XMLParser.IXMLRoot {
public string Type { get; } = "Register";
public string Prefix { get; } = "register";
public string[] XPathContainer { get; } = { ".//data//kommentare/kommcat", ".//kommentare/kommcat" };
public (string Key, string xPath, Func<XElement, string?> KeyFunc, bool Searchable)[]? XPathCollection { get; } = {
("comments-register", "/opus/data/kommentare/kommcat[@value='neuzeit']/kommentar", GetKey, true),
("comments-register", "/opus/kommentare/kommcat[@value='neuzeit']/kommentar", GetKey, true),
("comments-edition", "/opus/data/kommentare/kommcat[@value='editionen']/kommentar", GetKey, true),
("comments-edition", "/opus/kommentare/kommcat[@value='editionen']/kommentar", GetKey, true),
("comments-forschung", "/opus/data/kommentare/kommcat[@value='forschung']/kommentar", GetKey, true),
("comments-forschung", "/opus/kommentare/kommcat[@value='forschung']/kommentar", GetKey, true),
("comments-bibel", "/opus/data/kommentare/kommcat[@value='bibel']/kommentar", GetKey, false),
("comments-bibel", "/opus/kommentare/kommcat[@value='bibel']/kommentar", GetKey, false),
};
public Predicate<XElement> IsCollectedObject { get; } = (elem) => {
if (elem.Name == "kommentar") return true;
else return false;
};
public Func<XElement, string?> GetKey { get; } = (elem) => {
public static Func<XElement, string?> GetKey { get; } = (elem) => {
var index = elem.Attribute("id");
if (index != null && !String.IsNullOrWhiteSpace(index.Value))
return index.Value;

16
HaWeb/Settings/XMLRoots/DescriptionsRoot.cs
View File

@@ -7,18 +7,22 @@ public class DescriptionsRoot : HaWeb.XMLParser.IXMLRoot {
public string Type { get; } = "Metadaten";
public string Prefix { get; } = "metadaten";
public string[] XPathContainer { get; } = { ".//data/descriptions", ".//descriptions" };
public (string Key, string xPath, Func<XElement, string?> KeyFunc, bool Searchable)[]? XPathCollection { get; } = {
("metas", "/opus/descriptions/letterDesc", GetKey, false),
("metas", "/opus/data/descriptions/letterDesc", GetKey, false)
};
public Predicate<XElement> IsCollectedObject { get; } = (elem) => {
if (elem.Name == "letterDesc") return true;
return false;
};
// public Func<XElement, string?> GetKey { get; } = (elem) => {
// var index = elem.Attribute("ref");
// if (index != null && !String.IsNullOrWhiteSpace(index.Value))
// return index.Value;
// else return null;
// };
public static Func<XElement, string?> GetKey { get; } = (elem) => {
var index = elem.Attribute("ref");
if (index != null && !String.IsNullOrWhiteSpace(index.Value))
return index.Value;
return null;
};
public List<(string, string?)>? GenerateFields(XMLRootDocument document) {
return null;

6
HaWeb/Settings/XMLRoots/DocumentRoot.cs
View File

@@ -8,13 +8,17 @@ public class DocumentRoot : HaWeb.XMLParser.IXMLRoot {
public string Type { get; } = "Brieftext";
public string Prefix { get; } = "brieftext";
public string[] XPathContainer { get; } = { ".//data/document", ".//document" };
public (string Key, string xPath, Func<XElement, string?> KeyFunc, bool Searchable)[]? XPathCollection { get; } = {
("letters", "/opus/data/document/letterText", GetKey, true),
("letters", "/opus/document/letterText", GetKey, true)
};
public Predicate<XElement> IsCollectedObject { get; } = (elem) => {
if (elem.Name == "letterText") return true;
else return false;
};
public Func<XElement, string?> GetKey { get; } = (elem) => {
public static Func<XElement, string?> GetKey { get; } = (elem) => {
var index = elem.Attribute("index");
if (index != null && !String.IsNullOrWhiteSpace(index.Value))
return index.Value;

6
HaWeb/Settings/XMLRoots/EditsRoot.cs
View File

@@ -7,13 +7,17 @@ public class EditsRoot : HaWeb.XMLParser.IXMLRoot {
public string Type { get; } = "Texteingriffe";
public string Prefix { get; } = "texteingriffe";
public string[] XPathContainer { get; } = { ".//data/edits", ".//edits" };
public (string Key, string xPath, Func<XElement, string?> KeyFunc, bool Searchable)[]? XPathCollection { get; } = {
("edits", "/data/edits/editreason", GetKey, true),
("edits", "/edits/editreason", GetKey, true)
};
public Predicate<XElement> IsCollectedObject { get; } = (elem) => {
if (elem.Name == "editreason") return true;
else return false;
};
public Func<XElement, string?> GetKey { get; } = (elem) => {
public static Func<XElement, string?> GetKey { get; } = (elem) => {
var index = elem.Attribute("index");
if (index != null && !String.IsNullOrWhiteSpace(index.Value))
return index.Value;

6
HaWeb/Settings/XMLRoots/MarginalsRoot.cs
View File

@@ -7,13 +7,17 @@ public class MarginalsRoot : HaWeb.XMLParser.IXMLRoot {
public string Type { get; } = "Stellenkommentar";
public string Prefix { get; } = "stellenkommentar";
public string[] XPathContainer { get; } = { ".//data/marginalien", ".//marginalien" };
public (string Key, string xPath, Func<XElement, string?> KeyFunc, bool Searchable)[]? XPathCollection { get; } = {
("marginals", "/data/marginalien/marginal", GetKey, true),
("marginals", "/marginalien/marginal", GetKey, true)
};
public Predicate<XElement> IsCollectedObject { get; } = (elem) => {
if (elem.Name == "marginal") return true;
else return false;
};
public Func<XElement, string?> GetKey { get; } = (elem) => {
public static Func<XElement, string?> GetKey { get; } = (elem) => {
var index = elem.Attribute("index");
if (index != null && !String.IsNullOrWhiteSpace(index.Value))
return index.Value;

12
HaWeb/Settings/XMLRoots/ReferencesRoot.cs
View File

@@ -7,6 +7,14 @@ public class ReferencesRoot : HaWeb.XMLParser.IXMLRoot {
public string Type { get; } = "Personen / Orte";
public string Prefix { get; } = "personenorte";
public string[] XPathContainer { get; } = { ".//data/definitions", ".//definitions" };
public (string Key, string xPath, Func<XElement, string?> KeyFunc, bool Searchable)[]? XPathCollection { get; } = {
("person-definitions", "/opus/data/definitions/personDefs/personDef", GetKey, false),
("person-definitions", "/opus/definitions/personDefs/personDef", GetKey, false),
("hand-definitions", "/opus/data/definitions/handDefs/handDef", GetKey, false),
("hand-definitions", "/opus/definitions/handDefs/handDef", GetKey, false),
("location-definitions", "/opus/data/definitions/locationDefs/locationDef", GetKey, false),
("location-definitions", "/opus/definitions/locationDefs/locationDef", GetKey, false)
};
public Predicate<XElement> IsCollectedObject { get; } = (elem) => {
if (elem.Name == "personDefs" || elem.Name == "structureDefs" || elem.Name == "handDefs" || elem.Name == "locationDefs")
@@ -14,8 +22,8 @@ public class ReferencesRoot : HaWeb.XMLParser.IXMLRoot {
return false;
};
public Func<XElement, string?> GetKey { get; } = (elem) => {
return elem.Name.ToString();
public static Func<XElement, string?> GetKey { get; } = (elem) => {
return elem.Attribute("index") != null ? elem.Attribute("index")!.Value : null;
};
public List<(string, string?)>? GenerateFields(XMLRootDocument document) {

8
HaWeb/Settings/XMLRoots/TraditionsRoot.cs
View File

@@ -7,17 +7,21 @@ public class TraditionsRoot : HaWeb.XMLParser.IXMLRoot {
public string Type { get; } = "Überlieferung";
public string Prefix { get; } = "ueberlieferung";
public string[] XPathContainer { get; } = { ".//data/traditions", ".//traditions" };
public (string Key, string xPath, Func<XElement, string?> KeyFunc, bool Searchable)[]? XPathCollection { get; } = {
("tradition", "/opus/data/traditions/letterTradition", GetKey, true),
("tradition", "/opus/traditions/letterTradition", GetKey, true)
};
public Predicate<XElement> IsCollectedObject { get; } = (elem) => {
if (elem.Name == "letterTradition") return true;
else return false;
};
public Func<XElement, string?> GetKey { get; } = (elem) => {
public static Func<XElement, string?> GetKey { get; } = (elem) => {
var index = elem.Attribute("ref");
if (index != null && !String.IsNullOrWhiteSpace(index.Value))
return index.Value;
else return null;
return null;
};
public List<(string, string?)>? GenerateFields(XMLRootDocument document) {

10
HaWeb/XMLParser/IXMLRoot.cs
View File

@@ -13,7 +13,15 @@ public interface IXMLRoot {
// XPaths to determine if container is present
public abstract string[] XPathContainer { get; }
// Tag Name of child objects to be collected
// Collections of Elements to be created from this Root
// Key: the key under which the element(s) will be files
// xPath: the (absolute) XPath to the element(s)
// KeyFunc: How to extrect an identifier for the single element in the collection
// Searchable: Will the element be indexed for full-text-search?
public abstract (string Key, string xPath, Func<XElement, string?> KeyFunc, bool Searchable)[]? XPathCollection { get; }
// Determines child objects to be collected
// (deprecated see collections above; only used internally)
public abstract Predicate<XElement> IsCollectedObject { get; }
// Gets the Key of a collected object

5
HaWeb/XMLParser/IXMLService.cs
View File

@@ -2,12 +2,13 @@ namespace HaWeb.XMLParser;
using System.Xml.Linq;
using Microsoft.AspNetCore.Mvc.ModelBinding;
using HaWeb.Models;
using HaXMLReader.Interfaces;
public interface IXMLService {
public IXMLRoot? GetRoot(string name);
public List<IXMLRoot>? GetRootsList();
public Dictionary<string, IXMLRoot>? GetRootsDictionary();
public List<XMLRootDocument>? ProbeHamannFile(XDocument document, ModelStateDictionary ModelState);
public List<XMLRootDocument>? ProbeFile(XDocument document, ModelStateDictionary ModelState);
public Dictionary<string, FileList?>? GetUsedDictionary();
public XElement? MergeUsedDocuments(ModelStateDictionary ModelState);
public void Use(XMLRootDocument doc);
@@ -17,4 +18,6 @@ public interface IXMLService {
public void UnUse(string prefix);
public void UnUseProduction();
public void SetInProduction();
public void SetInProduction(XDocument document);
public List<(string Index, List<(string Page, string Line, string Preview)> Results)>? SearchCollection(string collection, string searchword, IReaderService reader);
}

65
HaWeb/XMLParser/XMLService.cs
View File

@@ -1,7 +1,13 @@
namespace HaWeb.XMLParser;
using System.Xml.Linq;
using System.Xml.XPath;
using Microsoft.AspNetCore.Mvc.ModelBinding;
using HaWeb.Models;
using HaWeb.SearchHelpers;
using System.Collections.Concurrent;
using System.Threading.Tasks;
using System.Text;
using HaXMLReader.Interfaces;
public class XMLService : IXMLService {
private Dictionary<string, FileList?>? _Used;
@@ -9,6 +15,9 @@ public class XMLService : IXMLService {
private Stack<Dictionary<string, FileList?>>? _InProduction;
private Dictionary<string, Dictionary<string, CollectedItem>> _collectedProduction;
private Dictionary<string, Dictionary<string, CollectedItem>> _collectedUsed;
public XMLService() {
// Getting all classes which implement IXMLRoot for possible document endpoints
var types = _GetAllTypesThatImplementInterface<IXMLRoot>().ToList();
@@ -50,9 +59,63 @@ public class XMLService : IXMLService {
_InProduction.Push(inProduction);
}
public void SetInProduction(XDocument document) {
if (document == null || _Roots == null) return;
var ret = new ConcurrentDictionary<string, ConcurrentDictionary<string, CollectedItem>>();
Parallel.ForEach(_Roots, (root) => {
if (root.Value.XPathCollection != null)
foreach (var coll in root.Value.XPathCollection) {
var elem = document.XPathSelectElements(coll.xPath);
if (elem != null && elem.Any()) {
if (!ret.ContainsKey(coll.Key))
ret[coll.Key] = new ConcurrentDictionary<string, CollectedItem>();
foreach(var e in elem) {
var k = coll.KeyFunc(e);
if (k != null) {
var searchtext = coll.Searchable ?
StringHelpers.NormalizeWhiteSpace(e.ToString(), ' ', false) :
null;
ret[coll.Key][k] = new CollectedItem(k, e, root.Value, coll.Key, searchtext);
}
}
}
}
});
_collectedProduction = ret.ToDictionary(x => x.Key, y => y.Value.ToDictionary(z => z.Key, f => f.Value, null), null);
}
public List<(string Index, List<(string Page, string Line, string Preview)> Results)>? SearchCollection(string collection, string searchword, IReaderService reader) {
if (!_collectedProduction.ContainsKey(collection)) return null;
var searchableObjects = _collectedProduction[collection];
var res = new ConcurrentBag<(string Index, List<(string Page, string Line, string preview)> Results)>();
var sw = StringHelpers.NormalizeWhiteSpace(searchword.Trim());
Parallel.ForEach(searchableObjects, (obj) => {
if (obj.Value.SearchText != null) {
var state = new SearchState(sw);
var rd = reader.RequestStringReader(obj.Value.SearchText);
var parser = new HaWeb.HTMLParser.LineXMLHelper<SearchState>(state, rd, new StringBuilder(), null, null, null, SearchRules.TextRules, SearchRules.WhitespaceRules);
rd.Read();
if (state.Results != null)
res.Add((
obj.Value.Index,
state.Results.Select(x => (
x.Page,
x.Line,
parser.Lines != null ?
parser.Lines
.Where(y => y.Page == x.Page && y.Line == x.Line)
.Select(x => x.Text)
.FirstOrDefault(string.Empty)
: ""
)).ToList()));
}
});
return res.ToList();
}
public void UnUseProduction() => this._InProduction = null;
public List<XMLRootDocument>? ProbeHamannFile(XDocument document, ModelStateDictionary ModelState) {
public List<XMLRootDocument>? ProbeFile(XDocument document, ModelStateDictionary ModelState) {
if (document.Root!.Name != "opus") {
ModelState.AddModelError("Error", "A valid Hamann-Docuemnt must begin with <opus>");
return null;