+Benutzer filter, u. Spalte

helpers/imports/common.go

@@ -0,0 +1,88 @@
+package imports
+
+import (
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+	"time"
+)
+
+const importDirName = "import"
+
+var importNamePattern = regexp.MustCompile(`^(\d{4})-(\d{2})(?:-(\d{2}))?-MUSENALM-(DATA|FILES)`)
+
+type ImportCandidate struct {
+	Path  string
+	IsZip bool
+}
+
+func FindLatestImport(kind string) (*ImportCandidate, error) {
+	dirEntries, err := os.ReadDir(importDirName)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	want := "MUSENALM-" + strings.ToUpper(kind)
+	type candidate struct {
+		name string
+		ts   time.Time
+	}
+	candidates := []candidate{}
+	for _, entry := range dirEntries {
+		name := entry.Name()
+		matches := importNamePattern.FindStringSubmatch(name)
+		if matches == nil {
+			continue
+		}
+		if !strings.Contains(strings.ToUpper(name), want) {
+			continue
+		}
+		year := matches[1]
+		month := matches[2]
+		day := matches[3]
+		if day == "" {
+			day = "01"
+		}
+		ts, err := time.Parse("2006-01-02", year+"-"+month+"-"+day)
+		if err != nil {
+			continue
+		}
+		candidates = append(candidates, candidate{name: name, ts: ts})
+	}
+
+	if len(candidates) == 0 {
+		return nil, nil
+	}
+
+	sort.Slice(candidates, func(i, j int) bool {
+		if candidates[i].ts.Equal(candidates[j].ts) {
+			return candidates[i].name < candidates[j].name
+		}
+		return candidates[i].ts.Before(candidates[j].ts)
+	})
+	latest := candidates[len(candidates)-1].name
+	path := filepath.Join(importDirName, latest)
+	info, err := os.Stat(path)
+	if err != nil {
+		return nil, err
+	}
+
+	if info.IsDir() {
+		return &ImportCandidate{Path: path, IsZip: false}, nil
+	}
+	if strings.HasSuffix(strings.ToLower(latest), ".zip") {
+		return &ImportCandidate{Path: path, IsZip: true}, nil
+	}
+
+	return &ImportCandidate{Path: path, IsZip: false}, nil
+}
+
+func HasImport(kind string) bool {
+	candidate, err := FindLatestImport(kind)
+	return err == nil && candidate != nil
+}

helpers/imports/data.go

@@ -0,0 +1,252 @@
+package imports
+
+import (
+	"archive/zip"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/pocketbase/dbx"
+	"github.com/pocketbase/pocketbase/core"
+)
+
+type tableFile struct {
+	Name string
+	Path string
+	Zip  *zip.File
+}
+
+type columnValue struct {
+	Value  string
+	IsNull bool
+}
+
+func ImportData(app core.App, candidate *ImportCandidate) error {
+	if candidate == nil {
+		return fmt.Errorf("no data import candidate found")
+	}
+
+	files, err := listDataFiles(candidate)
+	if err != nil {
+		return err
+	}
+	if len(files) == 0 {
+		return fmt.Errorf("no XML files found in data import")
+	}
+
+	tableOrder := make([]string, 0, len(files))
+	for _, file := range files {
+		tableOrder = append(tableOrder, file.Name)
+	}
+
+	return app.RunInTransaction(func(txApp core.App) error {
+		for i := len(tableOrder) - 1; i >= 0; i-- {
+			if shouldSkipImportTable(tableOrder[i]) {
+				continue
+			}
+			if _, err := txApp.DB().NewQuery("DELETE FROM " + quoteTableName(tableOrder[i])).Execute(); err != nil {
+				return err
+			}
+		}
+
+		for _, file := range files {
+			if shouldSkipImportTable(file.Name) {
+				continue
+			}
+
+			reader, closeFn, err := openTableReader(file)
+			if err != nil {
+				return err
+			}
+			if err := importTableXML(txApp, reader); err != nil {
+				closeFn()
+				return err
+			}
+			closeFn()
+		}
+
+		return nil
+	})
+}
+
+func listDataFiles(candidate *ImportCandidate) ([]tableFile, error) {
+	files := []tableFile{}
+	if candidate.IsZip {
+		reader, err := zip.OpenReader(candidate.Path)
+		if err != nil {
+			return nil, err
+		}
+		defer reader.Close()
+
+		for _, file := range reader.File {
+			if file.FileInfo().IsDir() || !strings.HasSuffix(strings.ToLower(file.Name), ".xml") {
+				continue
+			}
+			files = append(files, tableFile{
+				Name: strings.TrimSuffix(filepath.Base(file.Name), ".xml"),
+				Zip:  file,
+			})
+		}
+	} else {
+		entries, err := os.ReadDir(candidate.Path)
+		if err != nil {
+			return nil, err
+		}
+		for _, entry := range entries {
+			if entry.IsDir() {
+				continue
+			}
+			name := entry.Name()
+			if !strings.HasSuffix(strings.ToLower(name), ".xml") {
+				continue
+			}
+			files = append(files, tableFile{
+				Name: strings.TrimSuffix(name, ".xml"),
+				Path: filepath.Join(candidate.Path, name),
+			})
+		}
+	}
+
+	sort.Slice(files, func(i, j int) bool {
+		return files[i].Name < files[j].Name
+	})
+	return files, nil
+}
+
+func openTableReader(file tableFile) (io.ReadCloser, func(), error) {
+	if file.Zip != nil {
+		reader, err := file.Zip.Open()
+		if err != nil {
+			return nil, func() {}, err
+		}
+		return reader, func() { _ = reader.Close() }, nil
+	}
+
+	reader, err := os.Open(file.Path)
+	if err != nil {
+		return nil, func() {}, err
+	}
+	return reader, func() { _ = reader.Close() }, nil
+}
+
+func importTableXML(app core.App, reader io.Reader) error {
+	decoder := xml.NewDecoder(reader)
+	var tableName string
+	inRow := false
+	rowValues := map[string]columnValue{}
+
+	for {
+		token, err := decoder.Token()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+
+		switch t := token.(type) {
+		case xml.StartElement:
+			if t.Name.Local == "table" {
+				for _, attr := range t.Attr {
+					if attr.Name.Local == "name" {
+						tableName = strings.TrimSpace(attr.Value)
+						break
+					}
+				}
+				if tableName == "" {
+					return fmt.Errorf("missing table name in XML")
+				}
+				continue
+			}
+
+			if t.Name.Local == "row" {
+				inRow = true
+				rowValues = map[string]columnValue{}
+				continue
+			}
+
+			if inRow {
+				colName := t.Name.Local
+				isNull := hasNullAttr(t.Attr)
+				var text string
+				if err := decoder.DecodeElement(&text, &t); err != nil {
+					return err
+				}
+				rowValues[colName] = columnValue{
+					Value:  text,
+					IsNull: isNull,
+				}
+			}
+		case xml.EndElement:
+			if t.Name.Local == "row" && inRow {
+				if err := insertRow(app, tableName, rowValues); err != nil {
+					return err
+				}
+				inRow = false
+			}
+		}
+	}
+	return nil
+}
+
+func insertRow(app core.App, tableName string, row map[string]columnValue) error {
+	if len(row) == 0 {
+		return nil
+	}
+	cols := make([]string, 0, len(row))
+	placeholders := make([]string, 0, len(row))
+	params := dbx.Params{}
+
+	idx := 0
+	for col, value := range row {
+		cols = append(cols, quoteIdentifier(col))
+		paramName := fmt.Sprintf("p%d", idx)
+		placeholders = append(placeholders, "{:"+paramName+"}")
+		if value.IsNull {
+			params[paramName] = nil
+		} else {
+			params[paramName] = value.Value
+		}
+		idx++
+	}
+
+	query := "INSERT INTO " + quoteTableName(tableName) +
+		" (" + strings.Join(cols, ", ") + ") VALUES (" + strings.Join(placeholders, ", ") + ")"
+	_, err := app.DB().NewQuery(query).Bind(params).Execute()
+	return err
+}
+
+func hasNullAttr(attrs []xml.Attr) bool {
+	for _, attr := range attrs {
+		if attr.Name.Local == "null" && strings.EqualFold(attr.Value, "true") {
+			return true
+		}
+	}
+	return false
+}
+
+func shouldSkipImportTable(name string) bool {
+	if name == "" {
+		return true
+	}
+	if strings.HasPrefix(name, "_") {
+		return true
+	}
+	switch name {
+	case "schema_migrations":
+		return true
+	}
+	return false
+}
+
+func quoteTableName(name string) string {
+	return "`" + strings.ReplaceAll(name, "`", "``") + "`"
+}
+
+func quoteIdentifier(name string) string {
+	return "`" + strings.ReplaceAll(name, "`", "``") + "`"
+}

helpers/imports/files.go

@@ -0,0 +1,104 @@
+package imports
+
+import (
+	"archive/zip"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/pocketbase/pocketbase/core"
+)
+
+func ImportFiles(app core.App, candidate *ImportCandidate) error {
+	if candidate == nil {
+		return fmt.Errorf("no file import candidate found")
+	}
+
+	if candidate.IsZip {
+		return importFilesFromZip(app, candidate.Path)
+	}
+	return importFilesFromDir(app, candidate.Path)
+}
+
+func importFilesFromZip(app core.App, zipPath string) error {
+	reader, err := zip.OpenReader(zipPath)
+	if err != nil {
+		return err
+	}
+	defer reader.Close()
+
+	for _, file := range reader.File {
+		if file.FileInfo().IsDir() {
+			continue
+		}
+		reader, err := file.Open()
+		if err != nil {
+			return err
+		}
+		if err := importFileEntry(app, file.Name, reader); err != nil {
+			_ = reader.Close()
+			return err
+		}
+		_ = reader.Close()
+	}
+
+	return nil
+}
+
+func importFilesFromDir(app core.App, dirPath string) error {
+	return filepath.WalkDir(dirPath, func(path string, entry fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if entry.IsDir() {
+			return nil
+		}
+		relPath, err := filepath.Rel(dirPath, path)
+		if err != nil {
+			return err
+		}
+		reader, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer reader.Close()
+		return importFileEntry(app, relPath, reader)
+	})
+}
+
+func importFileEntry(app core.App, relPath string, reader io.Reader) error {
+	parts := strings.Split(filepath.ToSlash(relPath), "/")
+	if len(parts) < 3 {
+		return nil
+	}
+
+	collectionName := parts[0]
+	recordId := parts[1]
+	filename := parts[len(parts)-1]
+	if collectionName == "" || recordId == "" || filename == "" {
+		return nil
+	}
+
+	collection, err := app.FindCollectionByNameOrId(collectionName)
+	if err != nil || collection == nil {
+		return nil
+	}
+
+	destDir := filepath.Join(app.DataDir(), "storage", collection.Id, recordId)
+	if err := os.MkdirAll(destDir, 0o755); err != nil {
+		return err
+	}
+	destPath := filepath.Join(destDir, filename)
+
+	dest, err := os.Create(destPath)
+	if err != nil {
+		return err
+	}
+	defer dest.Close()
+
+	_, err = io.Copy(dest, reader)
+	return err
+}

helpers/imports/fts.go

@@ -0,0 +1,138 @@
+package imports
+
+import (
+	"github.com/Theodor-Springmann-Stiftung/musenalm/dbmodels"
+	"github.com/pocketbase/pocketbase/core"
+)
+
+func RebuildFTS(app core.App) error {
+	if err := dbmodels.DeleteFTS5Data(app); err != nil {
+		return err
+	}
+
+	places := []*dbmodels.Place{}
+	if err := app.RecordQuery(dbmodels.PLACES_TABLE).All(&places); err != nil {
+		return err
+	}
+	agents := []*dbmodels.Agent{}
+	if err := app.RecordQuery(dbmodels.AGENTS_TABLE).All(&agents); err != nil {
+		return err
+	}
+	series := []*dbmodels.Series{}
+	if err := app.RecordQuery(dbmodels.SERIES_TABLE).All(&series); err != nil {
+		return err
+	}
+	items := []*dbmodels.Item{}
+	if err := app.RecordQuery(dbmodels.ITEMS_TABLE).All(&items); err != nil {
+		return err
+	}
+	entries := []*dbmodels.Entry{}
+	if err := app.RecordQuery(dbmodels.ENTRIES_TABLE).All(&entries); err != nil {
+		return err
+	}
+	contents := []*dbmodels.Content{}
+	if err := app.RecordQuery(dbmodels.CONTENTS_TABLE).All(&contents); err != nil {
+		return err
+	}
+
+	entriesSeries := []*dbmodels.REntriesSeries{}
+	if err := app.RecordQuery(dbmodels.RelationTableName(dbmodels.ENTRIES_TABLE, dbmodels.SERIES_TABLE)).All(&entriesSeries); err != nil {
+		return err
+	}
+	entriesAgents := []*dbmodels.REntriesAgents{}
+	if err := app.RecordQuery(dbmodels.RelationTableName(dbmodels.ENTRIES_TABLE, dbmodels.AGENTS_TABLE)).All(&entriesAgents); err != nil {
+		return err
+	}
+	contentsAgents := []*dbmodels.RContentsAgents{}
+	if err := app.RecordQuery(dbmodels.RelationTableName(dbmodels.CONTENTS_TABLE, dbmodels.AGENTS_TABLE)).All(&contentsAgents); err != nil {
+		return err
+	}
+
+	placesById := map[string]*dbmodels.Place{}
+	for _, place := range places {
+		placesById[place.Id] = place
+	}
+	agentsById := map[string]*dbmodels.Agent{}
+	for _, agent := range agents {
+		agentsById[agent.Id] = agent
+	}
+	seriesById := map[string]*dbmodels.Series{}
+	for _, s := range series {
+		seriesById[s.Id] = s
+	}
+	entriesById := map[string]*dbmodels.Entry{}
+	for _, entry := range entries {
+		entriesById[entry.Id] = entry
+	}
+
+	entriesSeriesMap := map[string][]*dbmodels.Series{}
+	for _, rel := range entriesSeries {
+		if series := seriesById[rel.Series()]; series != nil {
+			entriesSeriesMap[rel.Entry()] = append(entriesSeriesMap[rel.Entry()], series)
+		}
+	}
+	entriesAgentsMap := map[string][]*dbmodels.Agent{}
+	for _, rel := range entriesAgents {
+		if agent := agentsById[rel.Agent()]; agent != nil {
+			entriesAgentsMap[rel.Entry()] = append(entriesAgentsMap[rel.Entry()], agent)
+		}
+	}
+	contentsAgentsMap := map[string][]*dbmodels.Agent{}
+	for _, rel := range contentsAgents {
+		if agent := agentsById[rel.Agent()]; agent != nil {
+			contentsAgentsMap[rel.Content()] = append(contentsAgentsMap[rel.Content()], agent)
+		}
+	}
+
+	qp := dbmodels.FTS5InsertQuery(app, dbmodels.PLACES_TABLE, dbmodels.PLACES_FTS5_FIELDS)
+	qa := dbmodels.FTS5InsertQuery(app, dbmodels.AGENTS_TABLE, dbmodels.AGENTS_FTS5_FIELDS)
+	qs := dbmodels.FTS5InsertQuery(app, dbmodels.SERIES_TABLE, dbmodels.SERIES_FTS5_FIELDS)
+	qi := dbmodels.FTS5InsertQuery(app, dbmodels.ITEMS_TABLE, dbmodels.ITEMS_FTS5_FIELDS)
+	qe := dbmodels.FTS5InsertQuery(app, dbmodels.ENTRIES_TABLE, dbmodels.ENTRIES_FTS5_FIELDS)
+	qc := dbmodels.FTS5InsertQuery(app, dbmodels.CONTENTS_TABLE, dbmodels.CONTENTS_FTS5_FIELDS)
+
+	for _, place := range places {
+		if err := dbmodels.BulkInsertFTS5Place(qp, place); err != nil {
+			return err
+		}
+	}
+	for _, agent := range agents {
+		if err := dbmodels.BulkInsertFTS5Agent(qa, agent); err != nil {
+			return err
+		}
+	}
+	for _, s := range series {
+		if err := dbmodels.BulkInsertFTS5Series(qs, s); err != nil {
+			return err
+		}
+	}
+	for _, item := range items {
+		if err := dbmodels.BulkInsertFTS5Item(qi, item); err != nil {
+			return err
+		}
+	}
+
+	for _, entry := range entries {
+		entryPlaces := []*dbmodels.Place{}
+		for _, placeId := range entry.Places() {
+			if place := placesById[placeId]; place != nil {
+				entryPlaces = append(entryPlaces, place)
+			}
+		}
+		entryAgents := entriesAgentsMap[entry.Id]
+		entrySeries := entriesSeriesMap[entry.Id]
+		if err := dbmodels.BulkInsertFTS5Entry(qe, entry, entryPlaces, entryAgents, entrySeries); err != nil {
+			return err
+		}
+	}
+
+	for _, content := range contents {
+		entry := entriesById[content.Entry()]
+		contentAgents := contentsAgentsMap[content.Id]
+		if err := dbmodels.BulkInsertFTS5Content(qc, content, entry, contentAgents); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}