orte provider

This commit is contained in:
Simon Martens
2025-09-25 15:01:26 +02:00
parent 6ddded953b
commit f90468085c
9 changed files with 573 additions and 8 deletions

View File

@@ -9,6 +9,7 @@ import (
"github.com/Theodor-Springmann-Stiftung/kgpz_web/helpers"
"github.com/Theodor-Springmann-Stiftung/kgpz_web/helpers/logging"
"github.com/Theodor-Springmann-Stiftung/kgpz_web/providers"
"github.com/Theodor-Springmann-Stiftung/kgpz_web/providers/geonames"
"github.com/Theodor-Springmann-Stiftung/kgpz_web/providers/gnd"
searchprovider "github.com/Theodor-Springmann-Stiftung/kgpz_web/providers/search"
"github.com/Theodor-Springmann-Stiftung/kgpz_web/providers/xmlprovider"
@@ -52,12 +53,13 @@ type KGPZ struct {
// So we need to prevent concurrent pulls and serializations
// This is what fsmu is for. IT IS NOT FOR SETTING Config, Repo. GND or Library.
// Those are only set once during initalization and construction.
fsmu sync.Mutex
Config *providers.ConfigProvider
Repo *providers.GitProvider
GND *gnd.GNDProvider
Library *xmlmodels.Library
Search *searchprovider.SearchProvider
fsmu sync.Mutex
Config *providers.ConfigProvider
Repo *providers.GitProvider
GND *gnd.GNDProvider
Geonames *geonames.GeonamesProvider
Library *xmlmodels.Library
Search *searchprovider.SearchProvider
}
func NewKGPZ(config *providers.ConfigProvider) (*KGPZ, error) {
@@ -116,6 +118,9 @@ func (k *KGPZ) Init() error {
if err := k.initGND(); err != nil {
logging.Error(err, "Error reading GND-Cache. Continuing.")
}
if err := k.initGeonames(); err != nil {
logging.Error(err, "Error reading Geonames-Cache. Continuing.")
}
if sp, err := searchprovider.NewSearchProvider(filepath.Join(k.Config.Config.BaseDIR, k.Config.SearchPath)); err != nil {
logging.Error(err, "Error initializing SearchProvider. Continuing without Search.")
@@ -141,6 +146,11 @@ func (k *KGPZ) initGND() error {
return k.GND.ReadCache(filepath.Join(k.Config.BaseDIR, k.Config.GNDPath))
}
func (k *KGPZ) initGeonames() error {
k.Geonames = geonames.NewGeonamesProvider()
return k.Geonames.ReadCache(filepath.Join(k.Config.BaseDIR, k.Config.GeoPath))
}
func (k *KGPZ) Routes(srv *fiber.App) error {
srv.Get("/", func(c *fiber.Ctx) error {
c.Redirect(INDEX_URL)
@@ -198,6 +208,7 @@ func (k *KGPZ) Funcs() map[string]interface{} {
e["GetIssue"] = k.Library.Issues.Item
e["GetPiece"] = k.Library.Pieces.Item
e["GetGND"] = k.GND.Person
e["GetGeonames"] = k.Geonames.Place
e["LookupPieces"] = k.Library.Pieces.ReverseLookup
e["LookupWorks"] = k.Library.Works.ReverseLookup
@@ -224,9 +235,18 @@ func (k *KGPZ) Enrich() error {
go func() {
k.fsmu.Lock()
defer k.fsmu.Unlock()
// Fetch GND data for agents
data := xmlmodels.AgentsIntoDataset(k.Library.Agents)
k.GND.FetchPersons(data)
k.GND.WriteCache(filepath.Join(k.Config.BaseDIR, k.Config.GNDPath))
// Fetch Geonames data for places
if k.Library.Places != nil {
placeData := xmlmodels.PlacesIntoDataset(k.Library.Places)
k.Geonames.FetchPlaces(placeData)
k.Geonames.WriteCache(filepath.Join(k.Config.BaseDIR, k.Config.GeoPath))
}
}()
return nil

View File

@@ -7,6 +7,13 @@ import (
func GetPlace(kgpz *xmlmodels.Library) fiber.Handler {
return func(c *fiber.Ctx) error {
return c.Render("/ort/", nil)
placeID := c.Params("place")
place := kgpz.Places.Item(placeID)
if place == nil {
return c.SendStatus(fiber.StatusNotFound)
}
return c.Render("/ort/", fiber.Map{"place": place})
}
}

View File

@@ -0,0 +1,266 @@
package geonames
import (
"encoding/json"
"errors"
"io"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"github.com/Theodor-Springmann-Stiftung/kgpz_web/helpers/logging"
)
const (
GEONAMES_API_URL = "http://api.geonames.org/getJSON"
GEONAMES_USERNAME = "theodorspringmans"
)
type GeonamesProvider struct {
// Mutex is for file reading & writing, not place map access
mu sync.Mutex
Places sync.Map
// INFO: this holds all errors that occurred during fetching
// and is used to prevent further fetches of the same place.
errmu sync.Mutex
errs map[string]int
}
func NewGeonamesProvider() *GeonamesProvider {
return &GeonamesProvider{
errs: make(map[string]int),
}
}
func (p *GeonamesProvider) ReadCache(folder string) error {
p.mu.Lock()
defer p.mu.Unlock()
if err := p.readPlaces(folder); err != nil {
return err
}
return nil
}
func (p *GeonamesProvider) readPlaces(folder string) error {
info, err := os.Stat(folder)
if os.IsNotExist(err) {
return os.MkdirAll(folder, 0755)
}
if err != nil || !info.IsDir() {
return err
}
files, err := filepath.Glob(filepath.Join(folder, "*.json"))
// TODO: try to recover by recreating the folder
if err != nil {
return err
}
wg := sync.WaitGroup{}
wg.Add(len(files))
for _, file := range files {
go func(file string) {
p.readPlace(file)
wg.Done()
}(file)
}
wg.Wait()
return nil
}
func (p *GeonamesProvider) readPlace(file string) {
place := Place{}
f, err := os.Open(file)
if err != nil {
logging.Error(err, "Error opening file for reading: "+file)
return
}
defer f.Close()
bytevalue, err := io.ReadAll(f)
if err != nil {
logging.Error(err, "Error reading file: "+file)
return
}
if err := json.Unmarshal(bytevalue, &place); err != nil {
logging.Error(err, "Error unmarshalling file:"+file)
return
}
if place.KGPZURL != "" {
p.Places.Store(place.KGPZURL, place)
return
}
}
func (p *GeonamesProvider) WriteCache(folder string) error {
p.mu.Lock()
defer p.mu.Unlock()
if err := p.writePlaces(folder); err != nil {
return err
}
return nil
}
// INFO: this writes all places to the cache folder
// We do that on every fetch, it's easier that way
func (p *GeonamesProvider) writePlaces(folder string) error {
info, err := os.Stat(folder)
if err == os.ErrNotExist {
return os.MkdirAll(folder, 0755)
}
if err != nil || !info.IsDir() {
return err
}
wg := sync.WaitGroup{}
p.Places.Range(func(key, value interface{}) bool {
wg.Add(1)
go func(key string, value Place) {
p.writePlace(folder, key, value)
wg.Done()
}(key.(string), value.(Place))
return true
})
wg.Wait()
return nil
}
// INFO: this overwrites any existing files
func (p *GeonamesProvider) writePlace(folder, id string, place Place) {
// JSON marshalling of the place and sanity check:
filepath := filepath.Join(folder, place.KGPZID+".json")
f, err := os.Create(filepath)
if err != nil {
logging.Error(err, "Error creating file for writing: "+id)
return
}
defer f.Close()
bytevalue, err := json.Marshal(place)
if err != nil {
logging.Error(err, "Error marshalling place: "+id)
return
}
if _, err := f.Write(bytevalue); err != nil {
logging.Error(err, "Error writing file: "+id)
return
}
}
func (p *GeonamesProvider) Place(id string) *Place {
place, ok := p.Places.Load(id)
if !ok {
return nil
}
plc := place.(Place)
return &plc
}
func (p *GeonamesProvider) FetchPlaces(places []GeonamesData) {
wg := sync.WaitGroup{}
for _, place := range places {
if place.ID == "" || place.Geonames == "" {
continue
}
// TODO: place already fetched; check for updates??
if _, ok := p.Places.Load(place.Geonames); ok {
continue
}
p.errmu.Lock()
if _, ok := p.errs[place.Geonames]; ok {
continue
}
p.errmu.Unlock()
wg.Add(1)
go func(place *GeonamesData) {
defer wg.Done()
p.fetchPlace(place.ID, place.Geonames)
}(&place)
}
wg.Wait()
}
func (p *GeonamesProvider) fetchPlace(ID, GeonamesURL string) {
SPLITURL := strings.Split(GeonamesURL, "/")
if len(SPLITURL) < 2 {
logging.Error(nil, "Error parsing Geonames ID from: "+GeonamesURL)
return
}
GeonamesID := SPLITURL[len(SPLITURL)-1]
requestURL := GEONAMES_API_URL + "?geonameId=" + GeonamesID + "&username=" + GEONAMES_USERNAME
logging.Debug("Fetching place: " + ID + " with URL: " + requestURL)
request, err := http.NewRequest("GET", requestURL, nil)
if err != nil {
logging.Error(err, "Error creating request: "+ID)
return
}
var response *http.Response
// INFO: we do 3 retries with increasing time between them
for i := 0; i < 3; i++ {
response, err = http.DefaultClient.Do(request)
if err == nil && response.StatusCode < 400 {
if i > 0 {
logging.Info("Successfully fetched place: " + ID + " after " + strconv.Itoa(i) + " retries")
}
break
}
time.Sleep(time.Duration(i+1) * time.Second)
logging.Error(err, "Retry fetching place: "+ID)
}
if err != nil {
logging.Error(err, "Error fetching place: "+ID)
return
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
if response.StatusCode < 500 {
p.errmu.Lock()
p.errs[GeonamesURL] = response.StatusCode
p.errmu.Unlock()
}
logging.Error(errors.New("Error fetching place: " + ID + " with status code: " + http.StatusText(response.StatusCode)))
return
}
body, err := io.ReadAll(response.Body)
if err != nil {
logging.Error(err, "Error reading response body: "+ID)
return
}
// For debug purposes: Write response body to file:
// os.WriteFile("geonames_responses/"+ID+".json", body, 0644)
geonamesPlace := Place{}
if err := json.Unmarshal(body, &geonamesPlace); err != nil {
logging.Error(err, "Error unmarshalling response body: "+ID)
return
}
geonamesPlace.KGPZID = ID
geonamesPlace.KGPZURL = GeonamesURL
p.Places.Store(GeonamesURL, geonamesPlace)
}

View File

@@ -0,0 +1,5 @@
package geonames
type GeonamesData struct {
ID, Geonames string
}

View File

@@ -0,0 +1,85 @@
package geonames
import (
"fmt"
)
type Place struct {
KGPZID string `json:"kgpzid"`
KGPZURL string `json:"kgpzurl"`
GeonameId int `json:"geonameId,omitempty"`
Name string `json:"name,omitempty"`
AsciiName string `json:"asciiName,omitempty"`
ToponymName string `json:"toponymName,omitempty"`
Lat string `json:"lat,omitempty"`
Lng string `json:"lng,omitempty"`
CountryName string `json:"countryName,omitempty"`
CountryCode string `json:"countryCode,omitempty"`
CountryId string `json:"countryId,omitempty"`
Population int `json:"population,omitempty"`
WikipediaURL string `json:"wikipediaURL,omitempty"`
Timezone Timezone `json:"timezone,omitempty"`
Bbox BoundingBox `json:"bbox,omitempty"`
Fcode string `json:"fcode,omitempty"`
FcodeName string `json:"fcodeName,omitempty"`
Fcl string `json:"fcl,omitempty"`
FclName string `json:"fclName,omitempty"`
ContinentCode string `json:"continentCode,omitempty"`
AdminName1 string `json:"adminName1,omitempty"`
AdminName2 string `json:"adminName2,omitempty"`
AdminName3 string `json:"adminName3,omitempty"`
AdminName4 string `json:"adminName4,omitempty"`
AdminName5 string `json:"adminName5,omitempty"`
AdminCode1 string `json:"adminCode1,omitempty"`
AdminCode2 string `json:"adminCode2,omitempty"`
AdminCode3 string `json:"adminCode3,omitempty"`
AdminCode4 string `json:"adminCode4,omitempty"`
AdminId1 string `json:"adminId1,omitempty"`
AdminId2 string `json:"adminId2,omitempty"`
AdminId3 string `json:"adminId3,omitempty"`
AdminId4 string `json:"adminId4,omitempty"`
AdminCodes1 AdminCodes1 `json:"adminCodes1,omitempty"`
AlternateNames []AlternateName `json:"alternateNames,omitempty"`
Astergdem int `json:"astergdem,omitempty"`
Srtm3 int `json:"srtm3,omitempty"`
}
type Timezone struct {
TimeZoneId string `json:"timeZoneId,omitempty"`
GmtOffset float64 `json:"gmtOffset,omitempty"`
DstOffset float64 `json:"dstOffset,omitempty"`
}
type BoundingBox struct {
East float64 `json:"east,omitempty"`
West float64 `json:"west,omitempty"`
North float64 `json:"north,omitempty"`
South float64 `json:"south,omitempty"`
AccuracyLevel int `json:"accuracyLevel,omitempty"`
}
type AdminCodes1 struct {
ISO3166_2 string `json:"ISO3166_2,omitempty"`
}
type AlternateName struct {
Name string `json:"name,omitempty"`
Lang string `json:"lang,omitempty"`
IsPreferredName bool `json:"isPreferredName,omitempty"`
IsShortName bool `json:"isShortName,omitempty"`
}
func (p Place) String() string {
return fmt.Sprintf("Place{KGPZID: %v, Name: %v, GeonameId: %v, CountryName: %v, Lat: %v, Lng: %v, Population: %v, WikipediaURL: %v}",
p.KGPZID, p.Name, p.GeonameId, p.CountryName, p.Lat, p.Lng, p.Population, p.WikipediaURL)
}
func (p Place) PlaceName() string {
if p.Name != "" {
return p.Name
}
if p.AsciiName != "" {
return p.AsciiName
}
return p.ToponymName
}

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,130 @@
{{ $place := .place }}
{{ if $place }}
{{ $geonames := GetGeonames $place.Geo }}
<div class="container mx-auto px-4 py-8">
<!-- Place Header -->
<div class="flex items-start justify-between gap-4 mb-6">
<div class="flex-1">
<!-- Large serif name with permalink -->
<div class="text-2xl font-serif font-bold mb-2 flex items-center gap-3">
<span>{{ index $place.Names 0 }}</span>
<a href="/ort/{{ $place.ID }}" class="text-gray-500 hover:text-blue-600 transition-colors no-underline" title="Permalink zu {{ index $place.Names 0 }}">
<i class="ri-link text-lg"></i>
</a>
</div>
<!-- Geographic Information from Geonames -->
{{ if ne $geonames nil }}
<div class="text-lg text-gray-800 mb-4">
<!-- Country and Administrative Info -->
{{ if ne $geonames.CountryName "" }}
<div class="mb-2">
{{ $geonames.CountryName }}
{{ if ne $geonames.AdminName1 "" }}
, {{ $geonames.AdminName1 }}
{{ end }}
{{ if and (ne $geonames.AdminName2 "") (ne $geonames.AdminName2 $geonames.AdminName1) }}
, {{ $geonames.AdminName2 }}
{{ end }}
</div>
{{ end }}
<!-- Coordinates -->
{{ if and (ne $geonames.Lat "") (ne $geonames.Lng "") }}
<div class="text-gray-600 text-base">
<i class="ri-map-pin-line"></i> {{ $geonames.Lat }}, {{ $geonames.Lng }}
</div>
{{ end }}
<!-- Population -->
{{ if gt $geonames.Population 0 }}
<div class="text-gray-600 text-base">
<i class="ri-group-line"></i> {{ $geonames.Population }} Einwohner
</div>
{{ end }}
</div>
{{ end }}
</div>
<!-- External link symbols on the right -->
<div class="flex gap-3 flex-shrink-0 items-center">
{{ if ne $geonames nil }}
<!-- Wikipedia link if available -->
{{ if ne $geonames.WikipediaURL "" }}
<a href="https://{{ $geonames.WikipediaURL }}" target="_blank" class="hover:opacity-80 transition-opacity" title="Wikipedia">
<img src="/assets/wikipedia.png" alt="Wikipedia" class="w-6 h-6">
</a>
{{ end }}
<!-- Geonames link -->
{{ if ne $place.Geo "" }}
<a href="{{ $place.Geo }}" target="_blank" class="hover:opacity-80 transition-opacity" title="Geonames">
<i class="ri-global-line text-2xl text-blue-600"></i>
</a>
{{ end }}
{{ end }}
</div>
</div>
<!-- Additional place details -->
{{ if ne $geonames nil }}
<div class="bg-gray-50 rounded-lg p-6">
<h3 class="text-lg font-semibold mb-4">Geografische Details</h3>
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
{{ if ne $geonames.Timezone.TimeZoneId "" }}
<div>
<strong>Zeitzone:</strong> {{ $geonames.Timezone.TimeZoneId }}
</div>
{{ end }}
{{ if ne $geonames.FcodeName "" }}
<div>
<strong>Typ:</strong> {{ $geonames.FcodeName }}
</div>
{{ end }}
{{ if ne (len $geonames.AlternateNames) 0 }}
<div class="md:col-span-2">
<strong>Alternative Namen:</strong>
<div class="flex flex-wrap gap-2 mt-2">
{{ range $i, $altName := $geonames.AlternateNames }}
{{ if lt $i 10 }}
{{ if ne $altName.Name "" }}
<span class="bg-blue-100 text-blue-800 px-2 py-1 rounded-md text-sm">
{{ $altName.Name }}
{{ if ne $altName.Lang "" }}
<span class="text-blue-600">({{ $altName.Lang }})</span>
{{ end }}
</span>
{{ end }}
{{ end }}
{{ end }}
</div>
</div>
{{ end }}
</div>
</div>
{{ end }}
<!-- Back Navigation -->
<div class="mt-8 pt-6 border-t border-gray-200">
<a href="javascript:history.back()" class="inline-flex items-center gap-2 text-blue-600 hover:text-blue-800 transition-colors">
<i class="ri-arrow-left-line"></i>
Zur<75>ck
</a>
</div>
</div>
{{ else }}
<div class="container mx-auto px-4 py-8">
<div class="text-center">
<h1 class="text-2xl font-bold text-gray-800 mb-4">Ort nicht gefunden</h1>
<p class="text-gray-600 mb-6">Der angeforderte Ort existiert nicht in unserer Datenbank.</p>
<a href="javascript:history.back()" class="inline-flex items-center gap-2 text-blue-600 hover:text-blue-800 transition-colors">
<i class="ri-arrow-left-line"></i>
Zur<75>ck
</a>
</div>
</div>
{{ end }}

View File

@@ -0,0 +1,41 @@
{{ $place := .place }}
{{ if $place }}
<title>{{ index $place.Names 0 }} - KGPZ</title>
<meta name="description" content="Informationen zu {{ index $place.Names 0 }} in der K<>nigsberger Gelehrten und Politischen Zeitung.">
<meta name="keywords" content="{{ index $place.Names 0 }}, Ort, KGPZ, K<>nigsberg, Zeitung">
<!-- Open Graph tags for social media -->
<meta property="og:title" content="{{ index $place.Names 0 }} - KGPZ">
<meta property="og:description" content="Informationen zu {{ index $place.Names 0 }} in der K<>nigsberger Gelehrten und Politischen Zeitung.">
<meta property="og:type" content="article">
<meta property="og:url" content="/ort/{{ $place.ID }}">
<!-- JSON-LD structured data -->
{{ $geonames := GetGeonames $place.Geo }}
{{ if ne $geonames nil }}
<script type="application/ld+json">
{
"@context": "https://schema.org",
"@type": "Place",
"name": "{{ index $place.Names 0 }}",
"description": "Historischer Ort erw<72>hnt in der K<>nigsberger Gelehrten und Politischen Zeitung",
{{ if and (ne $geonames.Lat "") (ne $geonames.Lng "") }}
"geo": {
"@type": "GeoCoordinates",
"latitude": {{ $geonames.Lat }},
"longitude": {{ $geonames.Lng }}
},
{{ end }}
{{ if ne $geonames.CountryName "" }}
"addressCountry": "{{ $geonames.CountryName }}",
{{ end }}
{{ if ne $place.Geo "" }}
"sameAs": "{{ $place.Geo }}"
{{ end }}
}
</script>
{{ end }}
{{ else }}
<title>Ort nicht gefunden - KGPZ</title>
<meta name="description" content="Der angeforderte Ort wurde nicht gefunden.">
{{ end }}

View File

@@ -2,6 +2,7 @@ package xmlmodels
import (
"github.com/Theodor-Springmann-Stiftung/kgpz_web/providers/gnd"
"github.com/Theodor-Springmann-Stiftung/kgpz_web/providers/geonames"
"github.com/Theodor-Springmann-Stiftung/kgpz_web/providers/xmlprovider"
)
@@ -14,3 +15,13 @@ func AgentsIntoDataset(provider *xmlprovider.XMLProvider[Agent]) []gnd.GNDData {
}
return data
}
func PlacesIntoDataset(provider *xmlprovider.XMLProvider[Place]) []geonames.GeonamesData {
provider.Lock()
defer provider.Unlock()
var data []geonames.GeonamesData
for _, place := range provider.Array {
data = append(data, geonames.GeonamesData{ID: place.ID, Geonames: place.Geo})
}
return data
}