Introduced basic structure

.gitignore

@@ -1,2 +1,5 @@
 KGPZ/
+data_git/
+cache_geo/
+cache_gnd/
 config.json

helpers/errors.go

@@ -0,0 +1,16 @@
+package helpers
+
+import (
+	"fmt"
+	"os"
+)
+
+func MaybePanic(err error, msg string) {
+	if err == nil {
+		return
+	}
+
+	fmt.Println(msg)
+	fmt.Println("Error: ", err)
+	os.Exit(1)
+}

helpers/logging.go

@@ -0,0 +1,20 @@
+package helpers
+
+import "fmt"
+
+func LogOnDebug[T fmt.Stringer](object T, msg string) {
+	if msg != "" {
+		fmt.Println(msg)
+	}
+	fmt.Println(object)
+}
+
+func LogOnErr[T fmt.Stringer](object T, err error, msg string) {
+	if err != nil {
+		if msg != "" {
+			fmt.Println(msg)
+		}
+		fmt.Println(object)
+		fmt.Println("Error: ", err)
+	}
+}

kgpz_web.go

@@ -1,14 +1,10 @@
 package main
 
 import (
-	"encoding/json"
-	"fmt"
 	"os"
-	"strings"
 
-	"githib.com/Theodor-Springmann-Stiftung/kgpz_web/models"
+	"githib.com/Theodor-Springmann-Stiftung/kgpz_web/helpers"
 	"githib.com/Theodor-Springmann-Stiftung/kgpz_web/providers"
-	"github.com/kelseyhightower/envconfig"
 )
 
 // 1. Check if folder exists
@@ -18,130 +14,65 @@ import (
 //		- Setup GitHub webhook if set
 // 3. Serialize XML DATA
 
-type Config struct {
+type KGPZ struct {
-	// At least one of these should be set
+	Config *providers.ConfigProvider
-	GitURL          string `json:"git_url" envconfig:"GIT_URL"`
+	Repo   *providers.GitProvider
-	GitBranch       string `json:"git_branch" envconfig:"GIT_BRANCH"`
-	FolderPath      string `json:"folder_path" envconfig:"FOLDER_PATH"`
-	GNDPath         string `json:"gnd_path" envconfig:"GND_PATH"`
-	GeoPath         string `json:"geo_path" envconfig:"GEO_PATH"`
-	WebHookEndpoint string `json:"webhook_endpoint" envconfig:"WEBHOOK_ENDPOINT"`
-	WebHookSecret   string `json:"webhook_secret" envconfig:"WEBHOOK_SECRET"`
-	Debug           bool   `json:"debug" envconfig:"DEBUG"`
 }
 
-// Implement stringer
+func NewKGPZ(config *providers.ConfigProvider) *KGPZ {
-func (c *Config) String() string {
+	if config == nil {
-	return fmt.Sprintf("GitURL: %s\nGitBranch: %s\nFolderPath: %s\nGNDPath: %s\nGeoPath: %s\nWebHookEndpoint: %s\nWebHookSecret: %s\n",
+		panic("ConfigProvider is nil")
-		c.GitURL, c.GitBranch, c.FolderPath, c.GNDPath, c.GeoPath, c.WebHookEndpoint, c.WebHookSecret)
-}
-
-func main() {
-	cfg := &Config{}
-	cfg = readSettingsFile(cfg, "config.dev.json")
-	cfg = readSettingsFile(cfg, "config.json")
-	cfg = readSettingsEnv(cfg)
-	cfg = readDefaults(cfg)
-
-	fmt.Println("Running with config:")
-	fmt.Println(cfg)
-
-	if cfg.FolderPath == "" {
-		panic("Folder path not set. Exiting.")
 	}
 
-	gp := providers.NewGitProvider(cfg.GitURL, cfg.FolderPath, cfg.GitBranch)
+	if err := config.Validate(); err != nil {
+		helpers.MaybePanic(err, "Error validating config")
+	}
+
+	return &KGPZ{Config: config}
+}
+
+func (k *KGPZ) IsDebug() bool {
+	return k.Config.Debug
+}
+
+func (k *KGPZ) InitRepo() {
+	gp := providers.NewGitProvider(k.Config.Config.GitURL, k.Config.Config.FolderPath, k.Config.Config.GitBranch)
 
 	// If folder exists try to pull, otherwise clone:
 	// TODO: there is no need to panic if clone can't be done, jus log the errors
 	// The code will panic if the XML data can't be parsed.
 	if gp != nil {
-		if _, err := os.Stat(cfg.FolderPath); os.IsNotExist(err) {
+		if _, err := os.Stat(k.Config.FolderPath); os.IsNotExist(err) {
 			err := gp.Clone()
 			if err != nil {
-				logOnErr(gp, err, "Error cloning repo")
+				helpers.LogOnErr(gp, err, "Error cloning repo")
 			}
 		} else {
 			err := gp.Pull()
 			if err != nil {
-				logOnErr(gp, err, "Error pulling repo")
+				helpers.LogOnErr(gp, err, "Error pulling repo")
 			}
 		}
 
 		if err := gp.Validate(); err != nil {
-			logOnErr(gp, err, "Error validating repo")
+			helpers.LogOnErr(gp, err, "Error validating repo")
 			gp = nil
 		}
 
-		if cfg.Debug && gp != nil {
+		if k.IsDebug() && gp != nil {
-			logOnDebug(gp, "GitProvider")
+			helpers.LogOnDebug(gp, "GitProvider")
 		}
 	}
 
-	// At his point we may or may not have a GitProvider
-
 }
 
-func maybePanic(err error, msg string) {
+func main() {
-	if err == nil {
+	cfg := providers.NewConfigProvider([]string{"config.dev.json", "config.json"})
-		return
+	if err := cfg.Read(); err != nil {
+		helpers.MaybePanic(err, "Error reading config")
 	}
 
-	fmt.Println(msg)
+	kgpz := NewKGPZ(cfg)
-	fmt.Println("Error: ", err)
+	kgpz.InitRepo()
-	os.Exit(1)
+
-}
-
-func readSettingsFile(cfg *Config, path string) *Config {
-	f, err := os.Open(path)
-	if err != nil {
-		fmt.Println("Error: ", err)
-		fmt.Println("Coudln't open ", path)
-		return cfg
-	}
-	defer f.Close()
-
-	dec := json.NewDecoder(f)
-	err = dec.Decode(cfg)
-	maybePanic(err, "Error decoding config.json")
-
-	return cfg
-}
-
-func readSettingsEnv(cfg *Config) *Config {
-	_ = envconfig.Process("KGPZ", cfg)
-	return cfg
-}
-
-func readDefaults(cfg *Config) *Config {
-	if strings.TrimSpace(cfg.FolderPath) == "" {
-		cfg.FolderPath = models.DEFAULT_GIT_DIR
-	}
-
-	if strings.TrimSpace(cfg.GNDPath) == "" {
-		cfg.GNDPath = models.DEFAULT_GND_DIR
-	}
-
-	if strings.TrimSpace(cfg.GeoPath) == "" {
-		cfg.GeoPath = models.DEFAULT_GEO_DIR
-	}
-
-	return cfg
-}
-
-func logOnDebug[T fmt.Stringer](object T, msg string) {
-	if msg != "" {
-		fmt.Println(msg)
-	}
-	fmt.Println(object)
-}
-
-func logOnErr[T fmt.Stringer](object T, err error, msg string) {
-	if err != nil {
-		if msg != "" {
-			fmt.Println(msg)
-		}
-		fmt.Println(object)
-		fmt.Println("Error: ", err)
-	}
 }

models/consts.go

@@ -1,8 +0,0 @@
-package models
-
-const (
-	// File paths
-	DEFAULT_GIT_DIR = "./KGPZ"
-	DEFAULT_GEO_DIR = "./cache_geo"
-	DEFAULT_GND_DIR = "./cache_gnd"
-)

models/notes.md

@@ -0,0 +1,5 @@
+# Notes on xml reading
+
+The models expect certain files and a certain file XML structure and throw quite often if the structure is not as expected.
+This is by design, as the models are supposed to be used in a controlled environment. Changes in the XML schema should be reflected in the models.
+All collections in the models should be kept thread-safe. 

providers/config.go

@@ -0,0 +1,100 @@
+package providers
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+
+	"githib.com/Theodor-Springmann-Stiftung/kgpz_web/helpers"
+	"github.com/kelseyhightower/envconfig"
+)
+
+// WARNING: this is not intended to be used in a multi-threaded environment
+// Instatiate this once on startup before any goroutines are started
+const (
+	DEFAULT_GIT_DIR = "data_git"
+	DEFAULT_GND_DIR = "cache_gnd"
+	DEFAULT_GEO_DIR = "cache_geo"
+)
+
+type ConfigProvider struct {
+	Files []string
+	*Config
+}
+
+type Config struct {
+	// At least one of these should be set
+	GitURL          string `json:"git_url" envconfig:"GIT_URL"`
+	GitBranch       string `json:"git_branch" envconfig:"GIT_BRANCH"`
+	FolderPath      string `json:"folder_path" envconfig:"FOLDER_PATH"`
+	GNDPath         string `json:"gnd_path" envconfig:"GND_PATH"`
+	GeoPath         string `json:"geo_path" envconfig:"GEO_PATH"`
+	WebHookEndpoint string `json:"webhook_endpoint" envconfig:"WEBHOOK_ENDPOINT"`
+	WebHookSecret   string `json:"webhook_secret" envconfig:"WEBHOOK_SECRET"`
+	Debug           bool   `json:"debug" envconfig:"DEBUG"`
+}
+
+func NewConfigProvider(files []string) *ConfigProvider {
+	return &ConfigProvider{Files: files}
+}
+
+func (c *ConfigProvider) Read() error {
+	c.Config = &Config{}
+	for _, file := range c.Files {
+		c.Config = readSettingsFile(c.Config, file)
+	}
+	c.Config = readSettingsEnv(c.Config)
+	c.Config = readDefaults(c.Config)
+	return nil
+}
+
+func (c *ConfigProvider) Validate() error {
+	if strings.TrimSpace(c.Config.FolderPath) == "" {
+		return fmt.Errorf("Folder path not set")
+	}
+	return nil
+}
+
+func readSettingsFile(cfg *Config, path string) *Config {
+	f, err := os.Open(path)
+	if err != nil {
+		fmt.Println("Error: ", err)
+		fmt.Println("Coudln't open ", path)
+		return cfg
+	}
+	defer f.Close()
+
+	dec := json.NewDecoder(f)
+	err = dec.Decode(cfg)
+	helpers.MaybePanic(err, "Error decoding config.json")
+
+	return cfg
+}
+
+func readSettingsEnv(cfg *Config) *Config {
+	_ = envconfig.Process("KGPZ", cfg)
+	return cfg
+}
+
+func readDefaults(cfg *Config) *Config {
+	if strings.TrimSpace(cfg.FolderPath) == "" {
+		cfg.FolderPath = DEFAULT_GIT_DIR
+	}
+
+	if strings.TrimSpace(cfg.GNDPath) == "" {
+		cfg.GNDPath = DEFAULT_GND_DIR
+	}
+
+	if strings.TrimSpace(cfg.GeoPath) == "" {
+		cfg.GeoPath = DEFAULT_GEO_DIR
+	}
+
+	return cfg
+}
+
+// Implement stringer
+func (c *Config) String() string {
+	return fmt.Sprintf("GitURL: %s\nGitBranch: %s\nFolderPath: %s\nGNDPath: %s\nGeoPath: %s\nWebHookEndpoint: %s\nWebHookSecret: %s\n",
+		c.GitURL, c.GitBranch, c.FolderPath, c.GNDPath, c.GeoPath, c.WebHookEndpoint, c.WebHookSecret)
+}

providers/git.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"sync"
 	"time"
 
 	"github.com/go-git/go-git/v5"
@@ -19,6 +20,8 @@ var InvalidStateError = errors.New("The GitProvider is not in a valid state. Fix
 // In case of success in either case it updates the commit hash and date and closes the repo again.
 // The Files are opened  and serialized by the FSProvider, which operates on the same file path.
 type GitProvider struct {
+	mu sync.Mutex
+
 	URL    string
 	Path   string
 	Branch string
@@ -34,6 +37,9 @@ func NewGitProvider(url string, path string, branch string) *GitProvider {
 }
 
 func (g *GitProvider) Pull() error {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+
 	branch := plumbing.NewBranchReferenceName(g.Branch)
 	repo, err := git.PlainOpen(g.Path)
 	if err != nil {
@@ -65,6 +71,9 @@ func (g *GitProvider) Pull() error {
 }
 
 func (g *GitProvider) Clone() error {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+
 	branch := plumbing.NewBranchReferenceName(g.Branch)
 
 	repo, err := git.PlainClone(g.Path, false, &git.CloneOptions{
@@ -118,7 +127,11 @@ func (g *GitProvider) setValues(repo *git.Repository) error {
 	return err
 }
 
+// WARNING: this expects the repo to be in a certain state and is intended to be used in tests.
 func (g *GitProvider) Read() error {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+
 	repo, err := git.PlainOpen(g.Path)
 	if err != nil {
 		return err
@@ -138,6 +151,9 @@ func (g *GitProvider) Read() error {
 }
 
 func (g *GitProvider) Validate() error {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+
 	if g.Commit == "" || g.Date.IsZero() {
 		return InvalidStateError
 	}