Speed up startup

2025-10-29 00:55:32 +00:00 · 2024-11-10 19:05:06 +01:00
parent a81e78c0fd
commit bdd4eeab26
14 changed files with 463 additions and 208 deletions
--- a/kgpz_web.go
+++ b/kgpz_web.go
@@ -1,8 +1,10 @@
 package main

 import (
+	"fmt"
 	"os"
 	"path/filepath"
+	"sync"

 	"githib.com/Theodor-Springmann-Stiftung/kgpz_web/helpers"
 	"githib.com/Theodor-Springmann-Stiftung/kgpz_web/providers"
@@ -10,24 +12,39 @@ import (

 // 1. Check if folder exists
 //		- If not, clone the repo, if possible or throw if error
-// 2. If folder exists, try to pull the repo, and if successful:
+// 2. If the folder exists, we try to serialize -- and spawn a goroutine to pull.
+//		Upon pulling, we read in the current state of the repository, even if it's up to date.
+//		-> If the repo was changed we execute a callback and parse again.
+//		-> If pulling fails, we retry after a certain amount of time.
+//		   Still we can continue if serialization proceeds.
+//		-> If serialization fails, we throw an error, log it. We try to pull in the background.
 //		- setup commit date & hash
 //		- Setup GitHub webhook if set
-// 3. Serialize XML DATA

 const (
 	AGENTS_PATH     = "XML/akteure.xml"
 	PLACES_PATH     = "XML/orte.xml"
 	WORKS_PATH      = "XML/werke.xml"
 	CATEGORIES_PATH = "XML/kategorien.xml"
+
+	ISSUES_DIR = "XML/stuecke/"
+	PIECES_DIR = "XML/beitraege/"
 )

+type Library struct {
+	smu        sync.Mutex
+	Agents     *providers.AgentProvider
+	Places     *providers.PlaceProvider
+	Works      *providers.WorkProvider
+	Categories *providers.CategoryProvider
+	Issues     *providers.IssueProvider
+	Pieces     *providers.PieceProvider
+}
+
 type KGPZ struct {
 	Config *providers.ConfigProvider
 	Repo   *providers.GitProvider
-	Agents *providers.AgentProvider
-	Places *providers.PlaceProvider
-	Works  *providers.WorkProvider
+	Library
 }

 func NewKGPZ(config *providers.ConfigProvider) *KGPZ {
@@ -46,86 +63,208 @@ func (k *KGPZ) IsDebug() bool {
 	return k.Config.Debug
 }

+func (k *KGPZ) Pull() {
+	go func(k *KGPZ) {
+		if k.Repo == nil {
+			return
+		}
+
+		err, changed := k.Repo.Pull()
+		if err != nil {
+			helpers.LogOnErr(&k.Repo, err, "Error pulling repo")
+		}
+
+		if changed {
+			if k.IsDebug() {
+				helpers.LogOnDebug(&k.Repo, "GitProvider changed")
+			}
+			// Locking is handled in Serialize()
+			k.Serialize()
+		}
+	}(k)
+}
+
 func (k *KGPZ) InitRepo() {
-	gp := providers.NewGitProvider(k.Config.Config.GitURL, k.Config.Config.FolderPath, k.Config.Config.GitBranch)
+	gp, err := providers.NewGitProvider(k.Config.Config.GitURL, k.Config.Config.FolderPath, k.Config.Config.GitBranch)
+	if err != nil {
+		helpers.LogOnErr(&gp, err, "Error creating GitProvider")
+		return
+	}

-	// If folder exists try to pull, otherwise clone:
-	// TODO: there is no need to panic if clone can't be done, jus log the errors
-	// The code will panic if the XML data can't be parsed.
-	if gp != nil {
-		if _, err := os.Stat(k.Config.FolderPath); os.IsNotExist(err) {
-			err := gp.Clone()
-			if err != nil {
-				helpers.LogOnErr(gp, err, "Error cloning repo")
-			}
-		} else {
-			err := gp.Pull()
-			if err != nil {
-				helpers.LogOnErr(gp, err, "Error pulling repo")
-			}
-		}
+	fmt.Println("InitRepo")
+	k.Repo = gp
+	k.Pull()

-		if err := gp.Validate(); err != nil {
-			helpers.LogOnErr(gp, err, "Error validating repo")
-			gp = nil
-		}
-
-		if k.IsDebug() && gp != nil {
-			helpers.LogOnDebug(gp, "GitProvider")
-		}
+	if k.IsDebug() {
+		helpers.LogOnDebug(&gp, "GitProvider")
 	}
 }

-func (k *KGPZ) InitAgents() {
+// This panics if the data cant be read, and there is no data read
+func (k *KGPZ) Serialize() {
+	k.smu.Lock()
+	defer k.smu.Unlock()
+	// TODO: maybe dont panic if a webhook can be setup, we need to check the requirements only when starting the server
+	// TODO: do this in parallel goroutines using a waitgroup
+	agents := k.InitAgents()
+	if agents == nil && k.Agents != nil {
+		helpers.LogOnErr(&k.Agents, nil, "Error initializing agents, keeping old state")
+	} else if agents == nil {
+		helpers.Panic(nil, "Error initializing agents")
+	} else {
+		k.Agents = agents
+	}
+
+	places := k.InitPlaces()
+	if places == nil && k.Places != nil {
+		helpers.LogOnErr(&k.Places, nil, "Error initializing places, keeping old state")
+	} else if places == nil {
+		helpers.Panic(nil, "Error initializing places")
+	} else {
+		k.Places = places
+	}
+
+	works := k.InitWorks()
+	if works == nil && k.Works != nil {
+		helpers.LogOnErr(&k.Works, nil, "Error initializing works, keeping old state")
+	} else if works == nil {
+		helpers.Panic(nil, "Error initializing works")
+	} else {
+		k.Works = works
+	}
+
+	categories := k.InitCategories()
+	if categories == nil && k.Categories != nil {
+		helpers.LogOnErr(&k.Categories, nil, "Error initializing categories, keeping old state")
+	} else if categories == nil {
+		helpers.Panic(nil, "Error initializing categories")
+	} else {
+		k.Categories = categories
+	}
+
+	issues := k.InitIssues()
+	if issues == nil && k.Issues != nil {
+		helpers.LogOnErr(&k.Issues, nil, "Error initializing issues, keeping old state")
+	} else if issues == nil {
+		helpers.Panic(nil, "Error initializing issues")
+	} else {
+		k.Issues = issues
+	}
+
+	pieces := k.InitPieces()
+	if pieces == nil && k.Pieces != nil {
+		helpers.LogOnErr(&k.Pieces, nil, "Error initializing pieces, keeping old state")
+	} else if pieces == nil {
+		helpers.Panic(nil, "Error initializing pieces")
+	} else {
+		k.Pieces = pieces
+	}
+}
+
+func (k *KGPZ) InitAgents() *providers.AgentProvider {
 	ap := providers.NewAgentProvider([]string{filepath.Join(k.Config.FolderPath, AGENTS_PATH)})
-	if ap != nil {
-		if err := ap.Load(); err != nil {
-			helpers.LogOnErr(ap, err, "Error loading agents")
-		}
-
-		if k.IsDebug() {
-			helpers.LogOnDebug(ap, "AgentProvider")
-		}
+	if err := ap.Load(); err != nil {
+		helpers.LogOnErr(&ap, err, "Error loading agents")
+		return nil
 	}
+
+	if k.IsDebug() {
+		helpers.LogOnDebug(&ap, "AgentProvider")
+	}
+
+	return ap
 }

-func (k *KGPZ) InitPlaces() {
+func (k *KGPZ) InitPlaces() *providers.PlaceProvider {
 	pp := providers.NewPlaceProvider([]string{filepath.Join(k.Config.FolderPath, PLACES_PATH)})
-	if pp != nil {
-		if err := pp.Load(); err != nil {
-			helpers.LogOnErr(pp, err, "Error loading places")
-		}
-
-		if k.IsDebug() {
-			helpers.LogOnDebug(pp, "PlaceProvider")
-		}
+	if err := pp.Load(); err != nil {
+		helpers.LogOnErr(&pp, err, "Error loading places")
+		return nil
 	}
+
+	if k.IsDebug() {
+		helpers.LogOnDebug(&pp, "PlaceProvider")
+	}
+
+	return pp
 }

-func (k *KGPZ) InitWorks() {
+func (k *KGPZ) InitWorks() *providers.WorkProvider {
 	wp := providers.NewWorkProvider([]string{filepath.Join(k.Config.FolderPath, WORKS_PATH)})
-	if wp != nil {
-		if err := wp.Load(); err != nil {
-			helpers.LogOnErr(wp, err, "Error loading works")
-		}
-
-		if k.IsDebug() {
-			helpers.LogOnDebug(wp, "WorkProvider")
-		}
+	if err := wp.Load(); err != nil {
+		helpers.LogOnErr(&wp, err, "Error loading works")
+		return nil
 	}
+
+	if k.IsDebug() {
+		helpers.LogOnDebug(&wp, "WorkProvider")
+	}
+
+	return wp
 }

-func (k *KGPZ) InitCategories() {
+func (k *KGPZ) InitCategories() *providers.CategoryProvider {
 	cp := providers.NewCategoryProvider([]string{filepath.Join(k.Config.FolderPath, CATEGORIES_PATH)})
-	if cp != nil {
-		if err := cp.Load(); err != nil {
-			helpers.LogOnErr(cp, err, "Error loading categories")
-		}
-
-		if k.IsDebug() {
-			helpers.LogOnDebug(cp, "CategoryProvider")
-		}
+	if err := cp.Load(); err != nil {
+		helpers.LogOnErr(&cp, err, "Error loading categories")
+		return nil
 	}
+
+	if k.IsDebug() {
+		helpers.LogOnDebug(&cp, "CategoryProvider")
+	}
+
+	return cp
+}
+
+func (k *KGPZ) InitIssues() *providers.IssueProvider {
+	files, err := getXMLFiles(filepath.Join(k.Config.FolderPath, ISSUES_DIR))
+
+	if err != nil {
+		helpers.MaybePanic(err, "Error getting issues files")
+	}
+
+	cp := providers.NewIssueProvider(*files)
+	if err := cp.Load(); err != nil {
+		helpers.LogOnErr(&cp, err, "Error loading issues")
+		return nil
+	}
+
+	if k.IsDebug() {
+		helpers.LogOnDebug(&cp, "IssueProvider")
+	}
+
+	return cp
+}
+
+func (k *KGPZ) InitPieces() *providers.PieceProvider {
+	files, err := getXMLFiles(filepath.Join(k.Config.FolderPath, PIECES_DIR))
+
+	if err != nil {
+		helpers.MaybePanic(err, "Error getting pieces files")
+		return nil
+	}
+
+	cp := providers.NewPieceProvider(*files)
+	if err := cp.Load(); err != nil {
+		helpers.LogOnErr(&cp, err, "Error loading pieces")
+	}
+
+	if k.IsDebug() {
+		helpers.LogOnDebug(&cp, "PieceProvider")
+	}
+
+	return cp
+}
+
+func getXMLFiles(path string) (*[]string, error) {
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		return nil, err
+	}
+
+	matches, err := filepath.Glob(filepath.Join(path, "*.xml"))
+
+	return &matches, err
 }

 func main() {
@@ -136,8 +275,5 @@ func main() {

 	kgpz := NewKGPZ(cfg)
 	kgpz.InitRepo()
-	kgpz.InitAgents()
-	kgpz.InitPlaces()
-	kgpz.InitWorks()
-	kgpz.InitCategories()
+	kgpz.Serialize()
 }