Some sync changes, graceful shutdwn

config.dev.json

@@ -3,5 +3,5 @@
 	"git_branch": "main",
 	"webhook_endpoint": "/webhook",
 	"webhook_secret": "secret",
-	"debug": false
+	"debug": true
 }

kgpz_web.go

@@ -3,8 +3,10 @@ package main
 import (
 	"fmt"
 	"os"
+	"os/signal"
 	"path/filepath"
 	"sync"
+	"syscall"
 
 	"githib.com/Theodor-Springmann-Stiftung/kgpz_web/helpers"
 	"githib.com/Theodor-Springmann-Stiftung/kgpz_web/providers"
@@ -32,7 +34,6 @@ const (
 )
 
 type Library struct {
-	smu        sync.Mutex
 	Agents     *providers.AgentProvider
 	Places     *providers.PlaceProvider
 	Works      *providers.WorkProvider
@@ -42,6 +43,7 @@ type Library struct {
 }
 
 type KGPZ struct {
+	lmu    sync.Mutex
 	Config *providers.ConfigProvider
 	Repo   *providers.GitProvider
 	Library
@@ -64,6 +66,8 @@ func (k *KGPZ) IsDebug() bool {
 }
 
 func (k *KGPZ) Pull() {
+	// TODO: what happens if the application quits mid-pull?
+	// We need to make sure to exit gracefully
 	go func(k *KGPZ) {
 		if k.Repo == nil {
 			return
@@ -102,63 +106,102 @@ func (k *KGPZ) InitRepo() {
 
 // This panics if the data cant be read, and there is no data read
 func (k *KGPZ) Serialize() {
-	k.smu.Lock()
-	defer k.smu.Unlock()
 	// TODO: maybe dont panic if a webhook can be setup, we need to check the requirements only when starting the server
-	// TODO: do this in parallel goroutines using a waitgroup
-	agents := k.InitAgents()
-	if agents == nil && k.Agents != nil {
-		helpers.LogOnErr(&k.Agents, nil, "Error initializing agents, keeping old state")
-	} else if agents == nil {
-		helpers.Panic(nil, "Error initializing agents")
-	} else {
-		k.Agents = agents
-	}
 
-	places := k.InitPlaces()
-	if places == nil && k.Places != nil {
-		helpers.LogOnErr(&k.Places, nil, "Error initializing places, keeping old state")
-	} else if places == nil {
-		helpers.Panic(nil, "Error initializing places")
-	} else {
-		k.Places = places
-	}
+	new := Library{}
 
-	works := k.InitWorks()
-	if works == nil && k.Works != nil {
-		helpers.LogOnErr(&k.Works, nil, "Error initializing works, keeping old state")
-	} else if works == nil {
-		helpers.Panic(nil, "Error initializing works")
-	} else {
-		k.Works = works
-	}
+	wg := sync.WaitGroup{}
+	wg.Add(6)
 
-	categories := k.InitCategories()
-	if categories == nil && k.Categories != nil {
-		helpers.LogOnErr(&k.Categories, nil, "Error initializing categories, keeping old state")
-	} else if categories == nil {
-		helpers.Panic(nil, "Error initializing categories")
-	} else {
-		k.Categories = categories
-	}
+	go func() {
+		defer wg.Done()
+		agents := k.InitAgents()
+		if agents == nil && k.Agents != nil {
+			helpers.LogOnErr(&k.Agents, nil, "Error initializing agents, keeping old state")
+			new.Agents = k.Agents
+			return
+		} else if agents == nil {
+			helpers.Panic(nil, "Error initializing agents")
+			return
+		}
+		new.Agents = agents
+	}()
 
-	issues := k.InitIssues()
-	if issues == nil && k.Issues != nil {
-		helpers.LogOnErr(&k.Issues, nil, "Error initializing issues, keeping old state")
-	} else if issues == nil {
-		helpers.Panic(nil, "Error initializing issues")
-	} else {
-		k.Issues = issues
-	}
+	go func() {
+		defer wg.Done()
+		places := k.InitPlaces()
+		if places == nil && k.Places != nil {
+			helpers.LogOnErr(&k.Places, nil, "Error initializing places, keeping old state")
+			new.Places = k.Places
+			return
+		} else if places == nil {
+			helpers.Panic(nil, "Error initializing places")
+			return
+		}
+		new.Places = places
+	}()
 
-	pieces := k.InitPieces()
-	if pieces == nil && k.Pieces != nil {
-		helpers.LogOnErr(&k.Pieces, nil, "Error initializing pieces, keeping old state")
-	} else if pieces == nil {
-		helpers.Panic(nil, "Error initializing pieces")
-	} else {
-		k.Pieces = pieces
-	}
+	go func() {
+		defer wg.Done()
+		works := k.InitWorks()
+		if works == nil && k.Works != nil {
+			helpers.LogOnErr(&k.Works, nil, "Error initializing works, keeping old state")
+			new.Works = k.Works
+			return
+		} else if works == nil {
+			helpers.Panic(nil, "Error initializing works")
+			return
+		}
+		new.Works = works
+	}()
+
+	go func() {
+		defer wg.Done()
+		categories := k.InitCategories()
+		if categories == nil && k.Categories != nil {
+			helpers.LogOnErr(&k.Categories, nil, "Error initializing categories, keeping old state")
+			new.Categories = k.Categories
+			return
+		} else if categories == nil {
+			helpers.Panic(nil, "Error initializing categories")
+			return
+		}
+		new.Categories = categories
+	}()
+
+	go func() {
+		defer wg.Done()
+		issues := k.InitIssues()
+		if issues == nil && k.Issues != nil {
+			helpers.LogOnErr(&k.Issues, nil, "Error initializing issues, keeping old state")
+			new.Issues = k.Issues
+			return
+		} else if issues == nil {
+			helpers.Panic(nil, "Error initializing issues")
+			return
+		}
+		new.Issues = issues
+	}()
+
+	go func() {
+		defer wg.Done()
+		pieces := k.InitPieces()
+		if pieces == nil && k.Pieces != nil {
+			helpers.LogOnErr(&k.Pieces, nil, "Error initializing pieces, keeping old state")
+			new.Pieces = k.Pieces
+			return
+		} else if pieces == nil {
+			helpers.Panic(nil, "Error initializing pieces")
+			return
+		}
+		new.Pieces = pieces
+	}()
+
+	wg.Wait()
+
+	k.lmu.Lock()
+	k.Library = new
+	k.lmu.Unlock()
 }
 
 func (k *KGPZ) InitAgents() *providers.AgentProvider {
@@ -257,6 +300,10 @@ func (k *KGPZ) InitPieces() *providers.PieceProvider {
 	return cp
 }
 
+func (k *KGPZ) Shutdown() {
+	k.Repo.Wait()
+}
+
 func getXMLFiles(path string) (*[]string, error) {
 	if _, err := os.Stat(path); os.IsNotExist(err) {
 		return nil, err
@@ -276,4 +323,23 @@ func main() {
 	kgpz := NewKGPZ(cfg)
 	kgpz.InitRepo()
 	kgpz.Serialize()
+
+	Cleanup(kgpz)
+}
+
+func Cleanup(k *KGPZ) {
+	sigs := make(chan os.Signal, 1)
+	done := make(chan bool, 1)
+
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
+
+	go func() {
+		_ = <-sigs
+		// INFO: here we can add a cleanup functions
+		k.Shutdown()
+		done <- true
+	}()
+
+	<-done
+	fmt.Println("Cleanup finished. Exiting.")
 }

providers/agents.go

@@ -15,6 +15,7 @@ type Agent struct {
 	SortName string   `xml:"sortiername"`
 	Life     string   `xml:"lebensdaten"`
 	GND      string   `xml:"gnd"`
+	Org      bool     `xml:"org,attr"`
 	Identifier
 	AnnotationNote
 }

providers/git.go

@@ -88,25 +88,28 @@ func (g *GitProvider) Pull() (error, bool) {
 		return err, false
 	}
 
-	if err := wt.Checkout(&git.CheckoutOptions{
-		Branch: branch,
-		Force:  true,
-	}); err != nil {
-		return err, false
-	}
-
 	if err := wt.Pull(&git.PullOptions{
 		RemoteName:    "origin",
 		ReferenceName: branch,
 		Progress:      os.Stdout,
-	}); err != nil && err != git.NoErrAlreadyUpToDate {
+	}); err != nil {
+		if err == git.NoErrAlreadyUpToDate {
+			return nil, false
+		}
 		return err, false
-	} else if err == git.NoErrAlreadyUpToDate {
-		return nil, false
 	}
 	defer wt.Clean(&git.CleanOptions{Dir: true})
 
-	return g.setValues(repo), true
+	oldCommit := g.Commit
+	if err := g.setValues(repo); err != nil {
+		return err, false
+	}
+
+	if oldCommit == g.Commit {
+		return nil, false
+	}
+
+	return nil, true
 }
 
 func (g *GitProvider) Clone() error {
@@ -233,6 +236,11 @@ func (g *GitProvider) ValidateBranch(repo *git.Repository) error {
 	return nil
 }
 
+func (g *GitProvider) Wait() {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+}
+
 func (g *GitProvider) ValidateCommit() error {
 	if g.Commit == "" || g.Date.IsZero() {
 		return InvalidStateError

providers/pieces.go

@@ -23,6 +23,7 @@ type Piece struct {
 	WorkRefs      []WorkRef       `xml:"werk"`
 	PieceRefs     []PieceRef      `xml:"beitrag"`
 	AdditionalRef []AdditionalRef `xml:"beilage"`
+	Datum         []KGPZDate      `xml:"datum"`
 	Incipit       []string        `xml:"incipit"`
 	Title         []string        `xml:"titel"`
 	Identifier

providers/xmlcommon.go

@@ -16,10 +16,11 @@ type URL struct {
 type AdditionalRef struct {
 	XMLName xml.Name `xml:"beilage"`
 	Reference
-	Datum string `xml:"datum,attr"`
-	Nr    string `xml:"nr,attr"`
-	Von   string `xml:"von,attr"`
-	Bis   string `xml:"bis,attr"`
+	Datum        string `xml:"datum,attr"`
+	Nr           string `xml:"nr,attr"`
+	AdditionalNo string `xml:"beilage,attr"`
+	Von          string `xml:"von,attr"`
+	Bis          string `xml:"bis,attr"`
 }
 
 type IssueRef struct {
@@ -49,6 +50,7 @@ type WorkRef struct {
 
 type PieceRef struct {
 	XMLName xml.Name `xml:"beitrag"`
+	Page    string   `xml:"s,attr"`
 	Reference
 }
 

providers/xmlprovider.go

@@ -20,22 +20,16 @@ type XMLProvider[T KGPZXML[T]] struct {
 }
 
 func (p *XMLProvider[T]) Load() error {
-	var wg sync.WaitGroup
 	for _, path := range p.paths {
-		wg.Add(1)
-		go func(path string) {
-			defer wg.Done()
-			var data T
-			if err := UnmarshalFile(path, &data); err != nil {
-				fmt.Println(err)
-				return
-			}
-			p.mu.Lock()
-			p.Items = p.Items.Append(data)
-			p.mu.Unlock()
-		}(path)
+		var data T
+		if err := UnmarshalFile(path, &data); err != nil {
+			fmt.Println(err)
+			return err
+		}
+		p.mu.Lock()
+		p.Items = p.Items.Append(data)
+		p.mu.Unlock()
 	}
-	wg.Wait()
 	return nil
 }