Setup Git Repository Parsing

HaWeb/XMLParser/IXMLInteractionService.cs

@@ -5,21 +5,17 @@ using HaWeb.Models;
 using HaDocument.Interfaces;
 using HaDocument.Models;
 using HaXMLReader.Interfaces;
+using Microsoft.Extensions.FileProviders;
 
-public interface IXMLService {
+public interface IXMLInteractionService {
+    public XElement? TryCreate();
+    public bool GetValidState();
+    public void Collect(List<IFileInfo> Files);
+    public Dictionary<string, FileList?>? GetLoaded();
     public IXMLRoot? GetRoot(string name);
     public List<IXMLRoot>? GetRootsList();
-    public Dictionary<string, IXMLRoot>? GetRootsDictionary();
-    public List<XMLRootDocument>? ProbeFile(XDocument document, ModelStateDictionary ModelState);
-    public Dictionary<string, FileList?>? GetUsedDictionary();
-    public XElement? MergeUsedDocuments(ModelStateDictionary ModelState);
-    public void Use(XMLRootDocument doc);
-    public void AutoUse(string prefix);
-    public void AutoUse(FileList filelist);
-    public Dictionary<string, FileList?>? GetInProduction();
-    public void UnUse(string prefix);
-    public void SetInProduction();
-    public void SetInProduction(XDocument document);
+    public void CreateSearchables(XDocument document);
+    public List<FileModel>? GetManagedFiles();
     public List<(string Index, List<(string Page, string Line, string Preview, string Identifier)> Results)>? SearchCollection(string collection, string searchword, IReaderService reader, ILibrary? lib);
     public List<(string Index, List<(string Page, string Line, string Preview, string Identifier)> Results)>? GetPreviews(List<(string, List<Marginal>)> places, IReaderService reader, ILibrary lib);
 }

HaWeb/XMLParser/IXMLRoot.cs

@@ -58,8 +58,5 @@ public interface IXMLRoot {
     //     });
     //     return ret;
     // }
-
-    public abstract XElement CreateHamannDocument(XElement element);
-
     public abstract void MergeIntoFile(XElement file, XMLRootDocument document);
 }

HaWeb/XMLParser/XMLInteractionService.cs

@@ -1,123 +1,169 @@
-namespace HaWeb.XMLParser;
+using System.Collections.Concurrent;
+using System.Diagnostics;
+using System.Text;
+using System.Xml;
 using System.Xml.Linq;
 using System.Xml.XPath;
-using Microsoft.AspNetCore.Mvc.ModelBinding;
-using HaWeb.Models;
-using HaWeb.SearchHelpers;
-using System.Collections.Concurrent;
-using System.Threading.Tasks;
-using System.Text;
-using HaXMLReader.Interfaces;
 using HaDocument.Interfaces;
 using HaDocument.Models;
+using HaWeb.FileHelpers;
+using HaWeb.Models;
+using HaWeb.SearchHelpers;
+using HaWeb.XMLParser;
 using HaWeb.XMLTests;
+using HaXMLReader.Interfaces;
+using Microsoft.Extensions.FileProviders;
 
-// XMLService provides a wrapper around the loaded and used XML data 
-public class XMLService : IXMLService {
-    private Dictionary<string, FileList?>? _Used;
-    private Dictionary<string, IXMLRoot>? _Roots;
-    private Dictionary<string, IXMLCollection>? _Collections;
+// Conditions for Successful create
+// All types there
+// Merging Success
+// Saving Success
+// Loading Success
 
-    private Stack<Dictionary<string, FileList?>>? _InProduction;
+// Startup (BEFORE IXMLFileProvider, After IHaDocumentWrapper)
+public class XMLInteractionService : IXMLInteractionService {
+    private readonly IXMLTestService _testService;
+    private readonly long _fileSizeLimit;
+    private readonly string[] _allowedExtensions = { ".xml" };
+    private readonly static XmlReaderSettings _xmlSettings = new XmlReaderSettings() {
+        CloseInput = true,
+        CheckCharacters = false,
+        ConformanceLevel = ConformanceLevel.Fragment,
+        IgnoreComments = true,
+        IgnoreProcessingInstructions = true,
+        IgnoreWhitespace = false
+    };
 
-    private Dictionary<string, ItemsCollection>? _collectedProduction;
-    private Dictionary<string, ItemsCollection>? _collectedUsed;
+    private Dictionary<string, IXMLRoot>? _RootDefs;
+    private Dictionary<string, IXMLCollection>? _CollectionDefs;
 
-    public XMLService() {
-        // Getting all classes which implement IXMLRoot for possible document endpoints
+    private List<FileModel>? _ManagedFiles;
+    private Dictionary<string, FileList?>? _Loaded;
+    private Dictionary<string, ItemsCollection>? _Collection;
+
+    private bool _ValidState = false;
+    
+    public XMLInteractionService(IConfiguration config, IXMLTestService testService) {
+        _testService = testService;
+        _fileSizeLimit = config.GetValue<long>("FileSizeLimit");
         var roottypes = _GetAllTypesThatImplementInterface<IXMLRoot>().ToList();
         roottypes.ForEach( x => {
-            if (this._Roots == null) this._Roots = new Dictionary<string, IXMLRoot>();
+            if (this._RootDefs == null) this._RootDefs = new Dictionary<string, IXMLRoot>();
             var instance = (IXMLRoot)Activator.CreateInstance(x)!;
-            if (instance != null) this._Roots.Add(instance.Prefix, instance);
+            if (instance != null) this._RootDefs.Add(instance.Prefix, instance);
         });
 
         var collectiontypes = _GetAllTypesThatImplementInterface<IXMLCollection>().ToList();
         collectiontypes.ForEach( x => {
-            if (this._Collections == null) this._Collections = new Dictionary<string, IXMLCollection>();
+            if (this._CollectionDefs == null) this._CollectionDefs = new Dictionary<string, IXMLCollection>();
             var instance = (IXMLCollection)Activator.CreateInstance(x)!;
-            if (instance != null && instance.IsGlobal()) this._Collections.Add(instance.Key, instance);
+            if (instance != null && instance.IsGlobal()) this._CollectionDefs.Add(instance.Key, instance);
         });
 
-        if (_Roots == null || !_Roots.Any())
+        if (_RootDefs == null || !_RootDefs.Any())
             throw new Exception("No classes for upload endpoints were found!");
 
-        if (_Collections == null || !_Collections.Any())
+        if (_CollectionDefs == null || !_CollectionDefs.Any())
             throw new Exception("No classes for object collection were found!");
     }
 
+    // Getters and Setters
+    public Dictionary<string, FileList?>? GetLoaded() => this._Loaded;
+
+    public List<FileModel>? GetManagedFiles() => this._ManagedFiles;
+
+    public List<IXMLRoot>? GetRootsList() => this._RootDefs == null ? null : this._RootDefs.Values.ToList();
+
+    public bool GetValidState() => this._ValidState;
+    
     public IXMLRoot? GetRoot(string name) {
-        if (_Roots == null) return null;
-        _Roots.TryGetValue(name, out var root);
+        if (_RootDefs == null) return null;
+        _RootDefs.TryGetValue(name, out var root);
         return root;
     }
 
-    public List<IXMLRoot>? GetRootsList() => this._Roots == null ? null : this._Roots.Values.ToList();
-
-    public Dictionary<string, IXMLRoot>? GetRootsDictionary() => this._Roots == null ? null : this._Roots;
-
-    public Dictionary<string, FileList?>? GetInProduction() { 
-        if (_InProduction == null) return null;
-        return this._InProduction.Peek();
-    }
-
-    public void SetInProduction() {
-        if (_Used == null) return;
-        var inProduction = new Dictionary<string, FileList?>();
-        foreach (var category in _Used) {
-            if (category.Value == null || category.Value.GetFileList() == null || !category.Value.GetFileList()!.Any()) 
-                return;
-            inProduction.Add(category.Key, category.Value);
-        }
-
-        if(_InProduction == null) _InProduction = new Stack<Dictionary<string, FileList?>>();
-        _InProduction.Push(inProduction);
-    }
-
-    public void SetInProduction(XDocument document) {
-        if (document == null || _Roots == null) return;
-        int numProcs = Environment.ProcessorCount;
-        int concurrencyLevel = numProcs * 2;
-        int startingSize = 2909;
-        int startingSizeAllCollections = 23;
-        var ret = new ConcurrentDictionary<string, ItemsCollection>(concurrencyLevel, startingSizeAllCollections);
-
-        if (_Collections != null)
-            Parallel.ForEach(_Collections, (coll) => { 
-                var elem = coll.Value.xPath.Aggregate(new List<XElement>(), (x, y) =>  { x.AddRange(document.XPathSelectElements(y).ToList()); return x; } );
-                if (elem != null && elem.Any()) {
-                    var items = new ConcurrentDictionary<string, CollectedItem>(concurrencyLevel, startingSize);
-                    foreach (var e in elem) {
-                        var k = coll.Value.GenerateKey(e);
-                        if (k != null) {
-                            var searchtext = coll.Value.Searchable ? 
-                                StringHelpers.NormalizeWhiteSpace(e.ToString(), ' ', false) : 
-                                null;
-                            var datafileds = coll.Value.GenerateDataFields != null ? 
-                                coll.Value.GenerateDataFields(e) :
-                                null;
-                            items[k] = new CollectedItem(k, e, coll.Value, datafileds, searchtext);
-                        }
-                    }
-                    if (items.Any()) {
-                        if (!ret.ContainsKey(coll.Key)) 
-                            ret[coll.Key] = new ItemsCollection(coll.Key, coll.Value);
-                        foreach (var item in items) 
-                            ret[coll.Key].Items.Add(item.Key, item.Value);
+    // Functions
+    public void Collect(List<IFileInfo> files) {
+        if (files == null || !files.Any()) return;
+        _ValidState = true;
+        List<FileModel> res = new List<FileModel>();
+        foreach (var f in files) {
+            var sb = new StringBuilder();
+            var m = _CreateFileModel(f, null);
+            res.Add(m);
+            // 1. Open File for Reading
+            try {
+                using (Stream file = f.CreateReadStream()) {
+                    // 2. Some security checks, if file empty, wrong start, wrong extension, too big
+                    if (!XMLFileHelpers.ProcessFile(file, f.Name, sb, _allowedExtensions, _fileSizeLimit)) {
+                        m.Log(sb.ToString());
+                        continue;
                     }
                 }
-            });
+            } catch {
+                m.Log( "Datei konnte nicht geöffnet werden.");
+                continue;
+            }
 
-        if (ret.Any()) {
-            Parallel.ForEach(ret, (collection) => {
-                collection.Value.GenerateGroupings();
-            });
+            // 3. Check validity of XML
+            try {
+                using (var xmlreader = XmlReader.Create(f.CreateReadStream(), _xmlSettings)) {
+                    var doc = XDocument.Load(xmlreader, LoadOptions.PreserveWhitespace | LoadOptions.SetLineInfo);
+
+                    // 4. Check if opus-Document
+                    // TODO: Unter der HOOD werden in ProbeFiles noch eigene Files gebaut!
+                    var docs = _ProbeFile(doc, m);
+                    if (docs == null || !docs.Any()) continue;
+
+                    // Success! File can be recognized and parsed.
+                    m.Validate();
+                    foreach (var d in docs) {
+                        if (_Loaded == null) _Loaded = new Dictionary<string, FileList?>();
+                        if (!_Loaded.ContainsKey(d.Prefix)) _Loaded.Add(d.Prefix, new FileList(d.XMLRoot));
+                        _Loaded[d.Prefix]!.Add(d);
+                    }
+                }
+            } catch (Exception ex) {
+                m.Log($"Ungültiges XML: {ex.Message}");
+                continue;
+            }
         }
-        _collectedProduction = ret.ToDictionary(x => x.Key, y => y.Value);
+        if(res.Any()) this._ManagedFiles = res;
+
+        // Set validity
+        foreach (var f in _ManagedFiles) {
+            if (!f.IsValid) _ValidState = false;
+            break;
+        }
+
+        // TODO: Speed up this:
+        var sw = new Stopwatch();
+        sw.Start();
+        _testService.Test(this);
+        sw.Stop();
+        Console.WriteLine("Syntaxcheck " + sw.ElapsedMilliseconds.ToString() + " ms");
+    }
+
+    public XElement? TryCreate() {
+        if (_Loaded == null || !_Loaded.Any() || _RootDefs == null || !_RootDefs.Any() || !_ValidState) return null;
+        var opus = new XElement("opus");
+        // TODO: Workaround for bug in HaDocument: roots have to be added in a specific order
+        var used = _Loaded.OrderByDescending(x => x.Key);
+        foreach (var category in used) {
+            if (category.Value == null || category.Value.GetFileList() == null || !category.Value.GetFileList()!.Any()) {
+                return null;
+            }
+            var documents = category.Value.GetFileList();
+            foreach (var document in documents!) {
+                document.XMLRoot.MergeIntoFile(opus, document);
+            }
+        }
+        return opus;
     }
 
     public List<(string Index, List<(string Page, string Line, string Preview, string Identifier)> Results)>? GetPreviews(List<(string, List<Marginal>)> places, IReaderService reader, ILibrary lib) {
-        var searchableObjects = _collectedProduction["letters"].Items;
+         if (!_Collection.ContainsKey("letters")) return null;
+        var searchableObjects = _Collection["letters"].Items;
         var res = new ConcurrentBag<(string Index, List<(string Page, string Line, string preview, string identifier)> Results)>();
 
         Parallel.ForEach(places, (obj) => {
@@ -148,8 +194,8 @@ public class XMLService : IXMLService {
     }
 
      public List<(string Index, List<(string Page, string Line, string Preview, string Identifier)> Results)>? SearchCollection(string collection, string searchword, IReaderService reader, ILibrary lib) {
-        if (!_collectedProduction.ContainsKey(collection)) return null;
-        var searchableObjects = _collectedProduction[collection].Items;
+        if (!_Collection.ContainsKey(collection)) return null;
+        var searchableObjects = _Collection[collection].Items;
         var res = new ConcurrentBag<(string Index, List<(string Page, string Line, string preview, string identifier)> Results)>();
         var sw = StringHelpers.NormalizeWhiteSpace(searchword.Trim());
 
@@ -202,110 +248,88 @@ public class XMLService : IXMLService {
         return res.ToList();
     }
 
-    public List<XMLRootDocument>? ProbeFile(XDocument document, ModelStateDictionary ModelState) {
-        if (document.Root!.Name != "opus") {
-            ModelState.AddModelError("Error", "A valid Hamann-Docuemnt must begin with <opus>");
-            return null;
-        }
+    public void CreateSearchables(XDocument document) {
+        if (document == null || _RootDefs == null) return;
+        int numProcs = Environment.ProcessorCount;
+        int concurrencyLevel = numProcs * 2;
+        int startingSize = 2909;
+        int startingSizeAllCollections = 23;
+        var ret = new ConcurrentDictionary<string, ItemsCollection>(concurrencyLevel, startingSizeAllCollections);
 
-        List<XMLRootDocument>? res = null;
-        if (document.Root != null && _Roots != null) {
-            foreach (var (_, root) in _Roots) {
-                var elements = root.IsTypeOf(document.Root);
-                if (elements != null &&  elements.Any())
-                    foreach (var elem in elements) {
-                        if (res == null) res = new List<XMLRootDocument>();
-                        res.Add(_createXMLRootDocument(root, elem));
+        if (_CollectionDefs != null)
+            Parallel.ForEach(_CollectionDefs, (coll) => { 
+                var elem = coll.Value.xPath.Aggregate(new List<XElement>(), (x, y) =>  { x.AddRange(document.XPathSelectElements(y).ToList()); return x; } );
+                if (elem != null && elem.Any()) {
+                    var items = new ConcurrentDictionary<string, CollectedItem>(concurrencyLevel, startingSize);
+                    foreach (var e in elem) {
+                        var k = coll.Value.GenerateKey(e);
+                        if (k != null) {
+                            var searchtext = coll.Value.Searchable ? 
+                                StringHelpers.NormalizeWhiteSpace(e.ToString(), ' ', false) : 
+                                null;
+                            var datafileds = coll.Value.GenerateDataFields != null ? 
+                                coll.Value.GenerateDataFields(e) :
+                                null;
+                            items[k] = new CollectedItem(k, e, coll.Value, datafileds, searchtext);
+                        }
+                    }
+                    if (items.Any()) {
+                        if (!ret.ContainsKey(coll.Key)) 
+                            ret[coll.Key] = new ItemsCollection(coll.Key, coll.Value);
+                        foreach (var item in items) 
+                            ret[coll.Key].Items.Add(item.Key, item.Value);
                     }
-            }
-        }
-        if (res == null) ModelState.AddModelError("Error", "Kein zum Hamann-Briefe-Projekt passendes XML gefunden.");
-        return res;
-    }
-
-    public Dictionary<string, FileList?>? GetUsedDictionary()
-        => this._Used;
-
-
-    // Adds a document and sets it to used
-    public void Use(XMLRootDocument doc) {
-        if (_Used == null) _Used = new Dictionary<string, FileList?>();
-        if (!_Used.ContainsKey(doc.Prefix)) _Used.Add(doc.Prefix, new FileList(doc.XMLRoot));
-        _Used[doc.Prefix]!.Add(doc);
-        _ = doc.GetElement();
-    }
-
-    public void UnUse(string prefix) {
-        if (_Used != null && _Used.ContainsKey(prefix)) {
-            // Unload the Elements so unused files don't use up the memory.
-            if (_Used[prefix]!.GetFileList() != null) {
-                foreach (var e in _Used[prefix]!.GetFileList()) {
-                    e.UnUse();
                 }
-            }
-            _Used.Remove(prefix);
+            });
+
+        if (ret.Any()) {
+            Parallel.ForEach(ret, (collection) => {
+                collection.Value.GenerateGroupings();
+            });
         }
-
-        return;
+        _Collection = ret.ToDictionary(x => x.Key, y => y.Value);
     }
 
-    // Performs detection of using on the specified document type
-    public void AutoUse(string prefix) {
-        if (_Used == null || !_Used.ContainsKey(prefix)) return;
-        AutoUse(_Used[prefix]!);
-    }
-
-    // Performs detection of using given a list of files
-    public void AutoUse(FileList filelist) {
-        FileList? res = null;
-        var list = filelist.GetFileList();
-        var prefix = filelist.XMLRoot.Prefix;
-
-        if (list == null) return;
-        if (_Used != null && _Used.ContainsKey(prefix)) _Used.Remove(prefix);
-
-        // TODO: Item1
-        var lookup = list.ToLookup(x => x.IdentificationString.Item2);
-        foreach (var idstring in lookup) {
-            var ordered = idstring.OrderBy(x => x.Date);
-            if (res == null) res = new FileList(filelist.XMLRoot);
-            Use(ordered.Last());
-        }
-    }
-
-    public XElement? MergeUsedDocuments(ModelStateDictionary ModelState) {
-        if (_Used == null || _Roots == null) {
-            ModelState.AddModelError("Error", "Keine Dokumente ausgewählt");
-            return null;
-        }
-
-        var opus = new XElement("opus");
-        // TODO: Workaround for bug in HaDocument: roots have to be added in a specific order
-        var used = _Used.OrderByDescending(x => x.Key);
-        foreach (var category in used) {
-            if (category.Value == null || category.Value.GetFileList() == null || !category.Value.GetFileList()!.Any()) {
-                ModelState.AddModelError("Error", _Roots![category.Key].Type + " nicht vorhanden.");
-                return null;
-            }
-            var documents = category.Value.GetFileList();
-            foreach (var document in documents!) {
-                document.XMLRoot.MergeIntoFile(opus, document);
-            }
-        }
-
-        return opus;
-    }
-
-    private XMLRootDocument _createXMLRootDocument(IXMLRoot Root, XElement element) {
-        var doc = new XMLRootDocument(Root, Root.Prefix, Root.GenerateIdentificationString(element), element);
-        doc.Fields = Root.GenerateFields(doc);
-        return doc;
-    }
-
-    private IEnumerable<Type> _GetAllTypesThatImplementInterface<T>()
-    {
+    private IEnumerable<Type> _GetAllTypesThatImplementInterface<T>() {
         return System.Reflection.Assembly.GetExecutingAssembly()
             .GetTypes()
             .Where(type => typeof(T).IsAssignableFrom(type) && !type.IsInterface);
     }
+
+    private List<XMLRootDocument>? _ProbeFile(XDocument document, FileModel file) {
+        if (document.Root!.Name != "opus") {
+            file.Log("Ein gültiges Dokument muss mit <opus> beginnen.");
+            return null;
+        }
+
+        List<XMLRootDocument>? res = null;
+        if (document.Root != null && _RootDefs != null) {
+            foreach (var (_, root) in _RootDefs) {
+                var elements = root.IsTypeOf(document.Root);
+                if (elements != null &&  elements.Any())
+                    foreach (var elem in elements) {
+                        if (res == null) res = new List<XMLRootDocument>();
+                        res.Add(_createXMLRootDocument(root, elem, file));
+                    }
+            }
+        }
+        if (res == null) file.Log("Dokumenten-Typ nicht erkannt.");
+        return res;
+    }
+
+    private XMLRootDocument _createXMLRootDocument(IXMLRoot Root, XElement element, FileModel file) {
+        var doc = new XMLRootDocument(Root, Root.Prefix, Root.GenerateIdentificationString(element), element, file);
+        doc.Fields = Root.GenerateFields(doc);
+        return doc;
+    }
+
+    private FileModel _CreateFileModel(IFileInfo file, string? message) {
+        var m = new FileModel(file.Name, file);
+        if (!String.IsNullOrWhiteSpace(message)) {
+            m.Log(message);
+        }
+        return m;
+    }
+
+    
 }