diff --git a/.gitignore b/.gitignore index 6cb11f8..95e9676 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ KGPZ/ +data_git/ +cache_geo/ +cache_gnd/ config.json diff --git a/helpers/errors.go b/helpers/errors.go new file mode 100644 index 0000000..0debfd9 --- /dev/null +++ b/helpers/errors.go @@ -0,0 +1,16 @@ +package helpers + +import ( + "fmt" + "os" +) + +func MaybePanic(err error, msg string) { + if err == nil { + return + } + + fmt.Println(msg) + fmt.Println("Error: ", err) + os.Exit(1) +} diff --git a/helpers/logging.go b/helpers/logging.go new file mode 100644 index 0000000..646351d --- /dev/null +++ b/helpers/logging.go @@ -0,0 +1,20 @@ +package helpers + +import "fmt" + +func LogOnDebug[T fmt.Stringer](object T, msg string) { + if msg != "" { + fmt.Println(msg) + } + fmt.Println(object) +} + +func LogOnErr[T fmt.Stringer](object T, err error, msg string) { + if err != nil { + if msg != "" { + fmt.Println(msg) + } + fmt.Println(object) + fmt.Println("Error: ", err) + } +} diff --git a/kgpz_web.go b/kgpz_web.go index dab5a5e..4734f16 100644 --- a/kgpz_web.go +++ b/kgpz_web.go @@ -1,14 +1,10 @@ package main import ( - "encoding/json" - "fmt" "os" - "strings" - "githib.com/Theodor-Springmann-Stiftung/kgpz_web/models" + "githib.com/Theodor-Springmann-Stiftung/kgpz_web/helpers" "githib.com/Theodor-Springmann-Stiftung/kgpz_web/providers" - "github.com/kelseyhightower/envconfig" ) // 1. Check if folder exists @@ -18,130 +14,65 @@ import ( // - Setup GitHub webhook if set // 3. Serialize XML DATA -type Config struct { - // At least one of these should be set - GitURL string `json:"git_url" envconfig:"GIT_URL"` - GitBranch string `json:"git_branch" envconfig:"GIT_BRANCH"` - FolderPath string `json:"folder_path" envconfig:"FOLDER_PATH"` - GNDPath string `json:"gnd_path" envconfig:"GND_PATH"` - GeoPath string `json:"geo_path" envconfig:"GEO_PATH"` - WebHookEndpoint string `json:"webhook_endpoint" envconfig:"WEBHOOK_ENDPOINT"` - WebHookSecret string `json:"webhook_secret" envconfig:"WEBHOOK_SECRET"` - Debug bool `json:"debug" envconfig:"DEBUG"` +type KGPZ struct { + Config *providers.ConfigProvider + Repo *providers.GitProvider } -// Implement stringer -func (c *Config) String() string { - return fmt.Sprintf("GitURL: %s\nGitBranch: %s\nFolderPath: %s\nGNDPath: %s\nGeoPath: %s\nWebHookEndpoint: %s\nWebHookSecret: %s\n", - c.GitURL, c.GitBranch, c.FolderPath, c.GNDPath, c.GeoPath, c.WebHookEndpoint, c.WebHookSecret) -} - -func main() { - cfg := &Config{} - cfg = readSettingsFile(cfg, "config.dev.json") - cfg = readSettingsFile(cfg, "config.json") - cfg = readSettingsEnv(cfg) - cfg = readDefaults(cfg) - - fmt.Println("Running with config:") - fmt.Println(cfg) - - if cfg.FolderPath == "" { - panic("Folder path not set. Exiting.") +func NewKGPZ(config *providers.ConfigProvider) *KGPZ { + if config == nil { + panic("ConfigProvider is nil") } - gp := providers.NewGitProvider(cfg.GitURL, cfg.FolderPath, cfg.GitBranch) + if err := config.Validate(); err != nil { + helpers.MaybePanic(err, "Error validating config") + } + + return &KGPZ{Config: config} +} + +func (k *KGPZ) IsDebug() bool { + return k.Config.Debug +} + +func (k *KGPZ) InitRepo() { + gp := providers.NewGitProvider(k.Config.Config.GitURL, k.Config.Config.FolderPath, k.Config.Config.GitBranch) // If folder exists try to pull, otherwise clone: // TODO: there is no need to panic if clone can't be done, jus log the errors // The code will panic if the XML data can't be parsed. if gp != nil { - if _, err := os.Stat(cfg.FolderPath); os.IsNotExist(err) { + if _, err := os.Stat(k.Config.FolderPath); os.IsNotExist(err) { err := gp.Clone() if err != nil { - logOnErr(gp, err, "Error cloning repo") + helpers.LogOnErr(gp, err, "Error cloning repo") } } else { err := gp.Pull() if err != nil { - logOnErr(gp, err, "Error pulling repo") + helpers.LogOnErr(gp, err, "Error pulling repo") } } if err := gp.Validate(); err != nil { - logOnErr(gp, err, "Error validating repo") + helpers.LogOnErr(gp, err, "Error validating repo") gp = nil } - if cfg.Debug && gp != nil { - logOnDebug(gp, "GitProvider") + if k.IsDebug() && gp != nil { + helpers.LogOnDebug(gp, "GitProvider") } } - // At his point we may or may not have a GitProvider - } -func maybePanic(err error, msg string) { - if err == nil { - return +func main() { + cfg := providers.NewConfigProvider([]string{"config.dev.json", "config.json"}) + if err := cfg.Read(); err != nil { + helpers.MaybePanic(err, "Error reading config") } - fmt.Println(msg) - fmt.Println("Error: ", err) - os.Exit(1) -} - -func readSettingsFile(cfg *Config, path string) *Config { - f, err := os.Open(path) - if err != nil { - fmt.Println("Error: ", err) - fmt.Println("Coudln't open ", path) - return cfg - } - defer f.Close() - - dec := json.NewDecoder(f) - err = dec.Decode(cfg) - maybePanic(err, "Error decoding config.json") - - return cfg -} - -func readSettingsEnv(cfg *Config) *Config { - _ = envconfig.Process("KGPZ", cfg) - return cfg -} - -func readDefaults(cfg *Config) *Config { - if strings.TrimSpace(cfg.FolderPath) == "" { - cfg.FolderPath = models.DEFAULT_GIT_DIR - } - - if strings.TrimSpace(cfg.GNDPath) == "" { - cfg.GNDPath = models.DEFAULT_GND_DIR - } - - if strings.TrimSpace(cfg.GeoPath) == "" { - cfg.GeoPath = models.DEFAULT_GEO_DIR - } - - return cfg -} - -func logOnDebug[T fmt.Stringer](object T, msg string) { - if msg != "" { - fmt.Println(msg) - } - fmt.Println(object) -} - -func logOnErr[T fmt.Stringer](object T, err error, msg string) { - if err != nil { - if msg != "" { - fmt.Println(msg) - } - fmt.Println(object) - fmt.Println("Error: ", err) - } + kgpz := NewKGPZ(cfg) + kgpz.InitRepo() + } diff --git a/models/consts.go b/models/consts.go deleted file mode 100644 index 6668c36..0000000 --- a/models/consts.go +++ /dev/null @@ -1,8 +0,0 @@ -package models - -const ( - // File paths - DEFAULT_GIT_DIR = "./KGPZ" - DEFAULT_GEO_DIR = "./cache_geo" - DEFAULT_GND_DIR = "./cache_gnd" -) diff --git a/models/notes.md b/models/notes.md new file mode 100644 index 0000000..5516282 --- /dev/null +++ b/models/notes.md @@ -0,0 +1,5 @@ +# Notes on xml reading + +The models expect certain files and a certain file XML structure and throw quite often if the structure is not as expected. +This is by design, as the models are supposed to be used in a controlled environment. Changes in the XML schema should be reflected in the models. +All collections in the models should be kept thread-safe. diff --git a/providers/config.go b/providers/config.go new file mode 100644 index 0000000..e4553a4 --- /dev/null +++ b/providers/config.go @@ -0,0 +1,100 @@ +package providers + +import ( + "encoding/json" + "fmt" + "os" + "strings" + + "githib.com/Theodor-Springmann-Stiftung/kgpz_web/helpers" + "github.com/kelseyhightower/envconfig" +) + +// WARNING: this is not intended to be used in a multi-threaded environment +// Instatiate this once on startup before any goroutines are started +const ( + DEFAULT_GIT_DIR = "data_git" + DEFAULT_GND_DIR = "cache_gnd" + DEFAULT_GEO_DIR = "cache_geo" +) + +type ConfigProvider struct { + Files []string + *Config +} + +type Config struct { + // At least one of these should be set + GitURL string `json:"git_url" envconfig:"GIT_URL"` + GitBranch string `json:"git_branch" envconfig:"GIT_BRANCH"` + FolderPath string `json:"folder_path" envconfig:"FOLDER_PATH"` + GNDPath string `json:"gnd_path" envconfig:"GND_PATH"` + GeoPath string `json:"geo_path" envconfig:"GEO_PATH"` + WebHookEndpoint string `json:"webhook_endpoint" envconfig:"WEBHOOK_ENDPOINT"` + WebHookSecret string `json:"webhook_secret" envconfig:"WEBHOOK_SECRET"` + Debug bool `json:"debug" envconfig:"DEBUG"` +} + +func NewConfigProvider(files []string) *ConfigProvider { + return &ConfigProvider{Files: files} +} + +func (c *ConfigProvider) Read() error { + c.Config = &Config{} + for _, file := range c.Files { + c.Config = readSettingsFile(c.Config, file) + } + c.Config = readSettingsEnv(c.Config) + c.Config = readDefaults(c.Config) + return nil +} + +func (c *ConfigProvider) Validate() error { + if strings.TrimSpace(c.Config.FolderPath) == "" { + return fmt.Errorf("Folder path not set") + } + return nil +} + +func readSettingsFile(cfg *Config, path string) *Config { + f, err := os.Open(path) + if err != nil { + fmt.Println("Error: ", err) + fmt.Println("Coudln't open ", path) + return cfg + } + defer f.Close() + + dec := json.NewDecoder(f) + err = dec.Decode(cfg) + helpers.MaybePanic(err, "Error decoding config.json") + + return cfg +} + +func readSettingsEnv(cfg *Config) *Config { + _ = envconfig.Process("KGPZ", cfg) + return cfg +} + +func readDefaults(cfg *Config) *Config { + if strings.TrimSpace(cfg.FolderPath) == "" { + cfg.FolderPath = DEFAULT_GIT_DIR + } + + if strings.TrimSpace(cfg.GNDPath) == "" { + cfg.GNDPath = DEFAULT_GND_DIR + } + + if strings.TrimSpace(cfg.GeoPath) == "" { + cfg.GeoPath = DEFAULT_GEO_DIR + } + + return cfg +} + +// Implement stringer +func (c *Config) String() string { + return fmt.Sprintf("GitURL: %s\nGitBranch: %s\nFolderPath: %s\nGNDPath: %s\nGeoPath: %s\nWebHookEndpoint: %s\nWebHookSecret: %s\n", + c.GitURL, c.GitBranch, c.FolderPath, c.GNDPath, c.GeoPath, c.WebHookEndpoint, c.WebHookSecret) +} diff --git a/providers/git.go b/providers/git.go index e7d4962..5d25ed8 100644 --- a/providers/git.go +++ b/providers/git.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "os" + "sync" "time" "github.com/go-git/go-git/v5" @@ -19,6 +20,8 @@ var InvalidStateError = errors.New("The GitProvider is not in a valid state. Fix // In case of success in either case it updates the commit hash and date and closes the repo again. // The Files are opened and serialized by the FSProvider, which operates on the same file path. type GitProvider struct { + mu sync.Mutex + URL string Path string Branch string @@ -34,6 +37,9 @@ func NewGitProvider(url string, path string, branch string) *GitProvider { } func (g *GitProvider) Pull() error { + g.mu.Lock() + defer g.mu.Unlock() + branch := plumbing.NewBranchReferenceName(g.Branch) repo, err := git.PlainOpen(g.Path) if err != nil { @@ -65,6 +71,9 @@ func (g *GitProvider) Pull() error { } func (g *GitProvider) Clone() error { + g.mu.Lock() + defer g.mu.Unlock() + branch := plumbing.NewBranchReferenceName(g.Branch) repo, err := git.PlainClone(g.Path, false, &git.CloneOptions{ @@ -118,7 +127,11 @@ func (g *GitProvider) setValues(repo *git.Repository) error { return err } +// WARNING: this expects the repo to be in a certain state and is intended to be used in tests. func (g *GitProvider) Read() error { + g.mu.Lock() + defer g.mu.Unlock() + repo, err := git.PlainOpen(g.Path) if err != nil { return err @@ -138,6 +151,9 @@ func (g *GitProvider) Read() error { } func (g *GitProvider) Validate() error { + g.mu.Lock() + defer g.mu.Unlock() + if g.Commit == "" || g.Date.IsZero() { return InvalidStateError }