mirror of
https://github.com/Theodor-Springmann-Stiftung/kgpz_web.git
synced 2025-10-29 09:05:30 +00:00
Improved logs for failed parses
This commit is contained in:
@@ -618,7 +618,9 @@ func (k *KGPZ) Pull() {
|
|||||||
|
|
||||||
if changed {
|
if changed {
|
||||||
logging.ObjDebug(&k.Repo, "Remote changed. Reparsing")
|
logging.ObjDebug(&k.Repo, "Remote changed. Reparsing")
|
||||||
k.Serialize()
|
if err := k.Serialize(); err != nil {
|
||||||
|
logging.Error(err, "Error parsing XML after git pull. Using mixed old/new data.")
|
||||||
|
}
|
||||||
k.EnrichAndRebuildIndex()
|
k.EnrichAndRebuildIndex()
|
||||||
|
|
||||||
// Rescan pictures after pull
|
// Rescan pictures after pull
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
package xmlprovider
|
package xmlprovider
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
"slices"
|
"slices"
|
||||||
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -96,12 +98,18 @@ func (p *XMLProvider[T]) Serialize(dataholder XMLRootElement[T], path string, la
|
|||||||
func (p *XMLProvider[T]) Cleanup(latest ParseMeta) {
|
func (p *XMLProvider[T]) Cleanup(latest ParseMeta) {
|
||||||
todelete := make([]string, 0)
|
todelete := make([]string, 0)
|
||||||
toappend := make([]*T, 0)
|
toappend := make([]*T, 0)
|
||||||
|
staleDataBySource := make(map[string][]string)
|
||||||
|
|
||||||
p.Infos.Range(func(key, value interface{}) bool {
|
p.Infos.Range(func(key, value interface{}) bool {
|
||||||
info := value.(ItemInfo)
|
info := value.(ItemInfo)
|
||||||
if !info.Parse.Equals(latest) {
|
if !info.Parse.Equals(latest) {
|
||||||
if !latest.Failed(info.Source) {
|
if !latest.Failed(info.Source) {
|
||||||
todelete = append(todelete, key.(string))
|
todelete = append(todelete, key.(string))
|
||||||
} else {
|
} else {
|
||||||
|
// Keep old data because the file failed to parse
|
||||||
|
keyStr := key.(string)
|
||||||
|
staleDataBySource[info.Source] = append(staleDataBySource[info.Source], keyStr)
|
||||||
|
|
||||||
item, ok := p.Items.Load(key)
|
item, ok := p.Items.Load(key)
|
||||||
if ok {
|
if ok {
|
||||||
i := item.(*T)
|
i := item.(*T)
|
||||||
@@ -114,6 +122,11 @@ func (p *XMLProvider[T]) Cleanup(latest ParseMeta) {
|
|||||||
return true
|
return true
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// Log stale data that is being kept
|
||||||
|
for source, ids := range staleDataBySource {
|
||||||
|
logging.Error(nil, "Keeping stale data from failed parse: "+source+" ("+fmt.Sprint(len(ids))+" items: "+strings.Join(ids, ", ")+")")
|
||||||
|
}
|
||||||
|
|
||||||
for _, key := range todelete {
|
for _, key := range todelete {
|
||||||
p.Infos.Delete(key)
|
p.Infos.Delete(key)
|
||||||
p.Items.Delete(key)
|
p.Items.Delete(key)
|
||||||
|
|||||||
Reference in New Issue
Block a user