wn/universal-data-ingest
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

changes for influxdb

Browse Source
This commit is contained in:
Wolfgang Hottgenroth
2026-02-04 14:58:13 +01:00
parent a78c6952f0
commit 97679561d8
5 changed files with 164 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
src/udi/database/abstract_database.go
View File

@@ -1,8 +1,6 @@
package database
import "time"
import "gorm.io/gorm"
type VariableType struct {
Label string `json:"label"`
@@ -13,35 +11,28 @@ type VariableType struct {
}
type Measurement struct {
Time time.Time `gorm:"not null;primary_key"`
Application string `gorm:"not null"`
Time time.Time
Application string
Device string
Attributes map[string]interface{} `gorm:"serializer:json;type:jsonb"`
Values map[string]VariableType `gorm:"serializer:json;type:jsonb"`
Attributes map[string]interface{}
Values map[string]VariableType
}
// Simplified structures for backward compatibility
type DeviceType struct {
Label string
ModelIdentifier string
Attributes map[string]interface{}
}
type Application struct {
gorm.Model
Label string `gorm:"not null"`
Attributes map[string]interface{} `gorm:"serializer:json;type:jsonb"`
}
type DeviceType struct {
gorm.Model
Label string `gorm:"not null"`
ModelIdentifier string
Attributes map[string]interface{} `gorm:"serializer:json;type:jsonb"`
Label string
Attributes map[string]interface{}
}
type Device struct {
gorm.Model
Label string `gorm:"not null;uniqueIndex:idx_label_application_id"`
ApplicationID int `gorm:"not null;uniqueIndex:idx_label_application_id"`
Label string
Application Application
DeviceTypeID int `gorm:"not null"`
DeviceType DeviceType
Attributes map[string]interface{} `gorm:"serializer:json;type:jsonb"`
Attributes map[string]interface{}
}

187
src/udi/database/database.go
View File

@@ -1,45 +1,146 @@
package database
import (
"log"
//"time"
"fmt"
"log"
"os"
"udi/counter"
"gorm.io/driver/postgres"
"gorm.io/gorm"
influxdb "github.com/influxdata/influxdb1-client/v2"
)
type DatabaseHandle struct {
initialized bool
dbh *gorm.DB
client influxdb.Client
database string
}
func NewDatabaseHandle() *DatabaseHandle {
var db DatabaseHandle
// inject the whole database configuration via the well-known PG* env variables
conn, err := gorm.Open(postgres.Open(""))
if err != nil {
log.Printf("Unable to open database connection: %s", err)
db.initialized = false
} else {
db.dbh = conn
db.initialized = true
log.Println("Database connection opened")
// Read configuration from environment variables
influxURL := os.Getenv("INFLUXDB_URL")
if influxURL == "" {
influxURL = "http://localhost:8086"
}
influxDB := os.Getenv("INFLUXDB_DATABASE")
if influxDB == "" {
influxDB = "udi"
}
username := os.Getenv("INFLUXDB_USER")
password := os.Getenv("INFLUXDB_PASSWORD")
// Create InfluxDB client
client, err := influxdb.NewHTTPClient(influxdb.HTTPConfig{
Addr: influxURL,
Username: username,
Password: password,
})
if err != nil {
log.Printf("Unable to create InfluxDB client: %s", err)
db.initialized = false
return &db
}
// Test connection
_, _, err = client.Ping(0)
if err != nil {
log.Printf("Unable to ping InfluxDB: %s", err)
db.initialized = false
client.Close()
return &db
}
db.client = client
db.database = influxDB
db.initialized = true
log.Printf("InfluxDB connection opened (URL: %s, Database: %s)", influxURL, influxDB)
return &db
}
func (self *DatabaseHandle) StoreMeasurement(measurement *Measurement) {
if !self.initialized {
log.Printf("Database connection not initialized, can not store, measurement %s lost", measurement)
log.Printf("Database connection not initialized, can not store, measurement %v lost", measurement)
counter.F("Stored")
return
}
result := self.dbh.Create(measurement)
if result.Error != nil {
log.Printf("Unable to insert, measurement %s lost, error: %s", measurement, result.Error)
// Create batch points
bp, err := influxdb.NewBatchPoints(influxdb.BatchPointsConfig{
Database: self.database,
Precision: "s",
})
if err != nil {
log.Printf("Unable to create batch points: %s", err)
counter.F("Stored")
return
}
// Build tags
tags := map[string]string{
"application": measurement.Application,
}
if measurement.Device != "" {
tags["device"] = measurement.Device
}
// Add attributes as tags
for key, value := range measurement.Attributes {
if strValue, ok := value.(string); ok {
tags[key] = strValue
} else {
tags[key] = fmt.Sprintf("%v", value)
}
}
// Build fields from Values
fields := make(map[string]interface{})
for key, varType := range measurement.Values {
// Store the value with the variable name as field key
fields[key] = varType.Value
// Optionally store metadata as separate fields
if varType.Unit != "" {
fields[key+"_unit"] = varType.Unit
}
if varType.Variable != "" {
fields[key+"_variable"] = varType.Variable
}
if varType.Status != "" {
fields[key+"_status"] = varType.Status
}
}
// Ensure we have at least one field
if len(fields) == 0 {
log.Printf("No fields to store in measurement, skipping")
counter.F("Stored")
return
}
// Create point
pt, err := influxdb.NewPoint(
"measurement",
tags,
fields,
measurement.Time,
)
if err != nil {
log.Printf("Unable to create point: %s", err)
counter.F("Stored")
return
}
bp.AddPoint(pt)
// Write batch
err = self.client.Write(bp)
if err != nil {
log.Printf("Unable to write to InfluxDB, measurement lost, error: %s", err)
counter.F("Stored")
return
}
@@ -48,49 +149,9 @@ func (self *DatabaseHandle) StoreMeasurement(measurement *Measurement) {
counter.S("Stored")
}
func (self *DatabaseHandle) GetDeviceByLabelAndApplication(applicationLabel string, deviceLabel string) (*Device, error) {
if ! self.initialized {
err := fmt.Errorf("Database connection not initialized")
return nil, err
func (self *DatabaseHandle) Close() {
if self.initialized && self.client != nil {
self.client.Close()
log.Println("InfluxDB connection closed")
}
var device Device
result := self.dbh.
Preload("Application").
Preload("DeviceType").
Joins("JOIN applications ON devices.application_id = applications.id").
Where("devices.label = ? AND applications.label = ?", deviceLabel, applicationLabel).
First(&device)
if result.Error != nil {
err := fmt.Errorf("Query failed: %s", result.Error)
return nil, err
}
return &device, nil
}
func (self *DatabaseHandle) GetDeviceByLabel(deviceLabel string) (*Device, error) {
if ! self.initialized {
err := fmt.Errorf("Database connection not initialized")
return nil, err
}
var device Device
result := self.dbh.
Preload("Application").
Preload("DeviceType").
Where("devices.label = ?", deviceLabel).
First(&device)
if result.Error != nil {
err := fmt.Errorf("Query failed: %s", result.Error)
return nil, err
}
return &device, nil
}

55
src/udi/database/migrate_database.go
View File

@@ -1,55 +0,0 @@
package database
import (
"log"
//"time"
"gorm.io/driver/postgres"
"gorm.io/gorm"
)
func Migrate() {
dsn := ""
db, err := gorm.Open(postgres.Open(dsn))
if err != nil {
log.Fatalf("Unable to open database connection: %s", err)
}
db.AutoMigrate(&Application{})
log.Println("Application created")
db.AutoMigrate(&DeviceType{})
log.Println("DeviceType created")
db.AutoMigrate(&Device{})
log.Println("Device created")
db.AutoMigrate(&Measurement{})
log.Println("Measurement created")
log.Println("Remember to call create_hypertable on measurements, sowhat I can't do that for you.")
/*
m := Measurement {
Time: time.Now(),
Application: "app",
Attributes: nil,
Values: []SensorType {
{ Variable: "Temperature", Unit: "Degree Celsius", Value: 1.0 },
{ Variable: "Temperature", Unit: "Degree Celsius", Value: 3.0 },
},
}
db.Create(&m)
m = Measurement {
Time: time.Now(),
Application: "app",
Attributes: nil,
Values: []SensorType {
{ Variable: "Temperature", Unit: "Degree Celsius", Value: 10.0 },
{ Variable: "Temperature", Unit: "Degree Celsius", Value: 30.0 },
},
}
db.Create(&m)
*/
}

11
src/udi/go.mod
View File

@@ -5,21 +5,12 @@ go 1.22.3
require (
github.com/eclipse/paho.mqtt.golang v1.5.0
github.com/google/uuid v1.6.0
github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c
github.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852
gorm.io/driver/postgres v1.5.11
gorm.io/gorm v1.25.12
)
require (
github.com/gorilla/websocket v1.5.3 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/pgx/v5 v5.7.2 // indirect
github.com/jackc/puddle/v2 v2.2.1 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
golang.org/x/crypto v0.32.0 // indirect
golang.org/x/net v0.34.0 // indirect
golang.org/x/sync v0.10.0 // indirect
golang.org/x/text v0.21.0 // indirect
)

18
src/udi/migrate_schema.go
View File

@@ -1,18 +0,0 @@
package main
import "log"
import "udi/database"
func main() {
log.SetPrefix("UDI Migrate Schema: ")
log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)
log.Println("Starting")
database.Migrate()
log.Println("Done")
}