wn/universal-data-ingest
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

dt1t and counter and refactoring using embedded interfaces
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
Details
ci/woodpecker/tag/woodpecker Pipeline was successful
Details

Browse Source
This commit is contained in:
Wolfgang Hottgenroth
2023-12-21 13:05:00 +01:00
parent 99d678b4b1
commit 8e6bea3f19
15 changed files with 276 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
src/udi/handlers/dt1t/dt1t.go
View File

@ -11,20 +11,18 @@ import (
)
var idSeq int = 0
type Dt1tHandler struct {
id int
handler.CommonHandler
ready bool
label string
dbh *database.DatabaseHandle
application string
device string
}
func NewDt1tHandler(config config.HandlerConfigT) handler.Handler {
func New(id string, config config.HandlerConfigT) handler.Handler {
t := &Dt1tHandler {
id: idSeq,
}
if config.Attributes["Application"] == "" {
@ -37,39 +35,29 @@ func NewDt1tHandler(config config.HandlerConfigT) handler.Handler {
return t
}
t.device = config.Attributes["Device"]
t.Id = id
idSeq += 1
t.dbh = database.NewDatabaseHandle()
t.ready = true
return t
}
func (self *Dt1tHandler) GetId() string {
return fmt.Sprintf("DT1T%d", self.id)
}
func lost(msg string, message handler.MessageT) {
log.Printf("Error: %s, message %s is lost", msg, message)
}
func (self *Dt1tHandler) Handle(message handler.MessageT) {
if ! self.ready {
log.Println("Handler is not marked as ready, message %s is lost", message)
self.Lost("Handler is not marked as ready", nil, message)
return
}
log.Printf("Handler DT1T %d processing %s -> %s", self.id, message.Topic, message.Payload)
// log.Printf("Handler DT1T %d processing %s -> %s", self.id, message.Topic, message.Payload)
temperature, err := strconv.Atoi(message.Payload)
if err != nil {
lost(fmt.Sprintf("Invalid raw value: %s", err), message)
self.Lost("Invalid raw value", err, message)
return
}
if temperature & 0x8000 != 0{
temperature = ((temperature - 1) ^ 0xffff) * -1
}
temperatureF := float32(temperature) / 10.0
log.Printf("TemperatureF: %f", temperatureF)
var measurement database.Measurement
measurement.Time = time.Now()
@ -84,8 +72,9 @@ func (self *Dt1tHandler) Handle(message handler.MessageT) {
measurement.Values = make(map[string]database.VariableType)
measurement.Values["Value"] = variable
log.Printf("Prepared measurement item: %s", measurement)
// log.Printf("Prepared measurement item: %s", measurement)
self.dbh.StoreMeasurement(&measurement)
self.S()
}

33
src/udi/handlers/handler/handler.go
View File

@ -1,6 +1,10 @@
package handler
import "time"
import (
"time"
"log"
"udi/counter"
)
type MessageT struct {
Timestamp time.Time
@ -11,5 +15,32 @@ type MessageT struct {
type Handler interface {
GetId() string
Handle(MessageT)
Lost(msg string, err error, message MessageT)
S()
F()
}
type CommonHandler struct {
Id string
}
func (self *CommonHandler) S() {
counter.SH(self.Id)
}
func (self *CommonHandler) F() {
counter.FH(self.Id)
}
func (self *CommonHandler) GetId() string {
return self.Id
}
func (self *CommonHandler) Lost(msg string, err error, message MessageT) {
if err != nil {
log.Printf("Error: %s, message %s is lost", msg, message)
} else {
log.Printf("Error: %s (%s), message %s is lost", msg, err, message)
}
self.F()
}

16
src/udi/handlers/iot/iot.go
View File

@ -1,29 +1,23 @@
package iot
import "log"
import "fmt"
import "udi/config"
import "udi/handlers/handler"
var idSeq int = 0
type IoTHandler struct {
id int
handler.CommonHandler
}
func NewIoTHandler() handler.Handler {
func New(id string, config config.HandlerConfigT) handler.Handler {
t := &IoTHandler {
id: idSeq,
}
idSeq += 1
t.Id = id
return t
}
func (self *IoTHandler) GetId() string {
return fmt.Sprintf("IoT%d", self.id)
}
func (self *IoTHandler) Handle(message handler.MessageT) {
log.Printf("Handler IoT %d processing %s -> %s", self.id, message.Topic, message.Payload)
log.Printf("Handler IoT %d processing %s -> %s", self.Id, message.Topic, message.Payload)
}

23
src/udi/handlers/mbgw3/mbgw3.go
View File

@ -1,11 +1,8 @@
package mbgw3
import (
"log"
//"reflect"
"time"
"strconv"
"fmt"
"encoding/json"
"udi/config"
"udi/handlers/handler"
@ -13,10 +10,8 @@ import (
)
var idSeq int = 0
type Mbgw3Handler struct {
id int
handler.CommonHandler
dbh *database.DatabaseHandle
}
@ -31,26 +26,21 @@ type Observation struct {
}
func NewMbgw3Handler(config config.HandlerConfigT) handler.Handler {
func New(id string, config config.HandlerConfigT) handler.Handler {
t := &Mbgw3Handler {
id: idSeq,
}
idSeq += 1
t.Id = id
t.dbh = database.NewDatabaseHandle()
return t
}
func (self *Mbgw3Handler) GetId() string {
return fmt.Sprintf("MBGW3%d", self.id)
}
func (self *Mbgw3Handler) Handle(message handler.MessageT) {
// log.Printf("Handler MBGW3 %d processing %s -> %s", self.id, message.Topic, message.Payload)
//log.Printf("Handler MBGW3 %d processing %s -> %s", self.Id, message.Topic, message.Payload)
var observation Observation
err := json.Unmarshal([]byte(message.Payload), &observation)
if err != nil {
log.Printf("Unable to parse payload into Observation struct, message %s -> %s is lost, error ", message.Topic, message.Payload, err)
self.Lost("Unable to parse payload into Observation struct", err, message)
return
}
@ -95,9 +85,10 @@ func (self *Mbgw3Handler) Handle(message handler.MessageT) {
}
}
// log.Printf("Prepared measurement item: %s", measurement)
//log.Printf("Prepared measurement item: %s", measurement)
self.dbh.StoreMeasurement(&measurement)
self.S()
}

17
src/udi/handlers/pv/pv.go
View File

@ -1,10 +1,8 @@
package pv
import (
"log"
"reflect"
"time"
"fmt"
"encoding/json"
"udi/config"
"udi/handlers/handler"
@ -12,10 +10,8 @@ import (
)
var idSeq int = 0
type PvHandler struct {
id int
handler.CommonHandler
dbh *database.DatabaseHandle
}
@ -40,18 +36,14 @@ type PvValue struct {
}
func NewPvHandler(config config.HandlerConfigT) handler.Handler {
func New(id string, config config.HandlerConfigT) handler.Handler {
t := &PvHandler {
id: idSeq,
}
idSeq += 1
t.Id = id
t.dbh = database.NewDatabaseHandle()
return t
}
func (self *PvHandler) GetId() string {
return fmt.Sprintf("PV%d", self.id)
}
func (self *PvHandler) Handle(message handler.MessageT) {
//log.Printf("Handler PV %d processing %s -> %s", self.id, message.Topic, message.Payload)
@ -59,7 +51,7 @@ func (self *PvHandler) Handle(message handler.MessageT) {
var pvValue PvValue
err := json.Unmarshal([]byte(message.Payload), &pvValue)
if err != nil {
log.Printf("Unable to parse payload into pvValue struct, message %s -> %s is lost, error: %s", message.Topic, message.Payload, err)
self.Lost("Unable to parse payload into pvValue struct", err, message)
return
}
@ -85,6 +77,7 @@ func (self *PvHandler) Handle(message handler.MessageT) {
}
self.dbh.StoreMeasurement(&measurement)
self.S()
}

32
src/udi/handlers/svej/svej.go
View File

@ -13,10 +13,8 @@ import (
"udi/database"
)
var idSeq int = 0
type SingleValueExtractorJsonpathHandler struct {
id int
handler.CommonHandler
ready bool
application string
deviceSelector string
@ -36,12 +34,10 @@ C:ConstantValue
*/
func NewSvejHandler(config config.HandlerConfigT) handler.Handler {
func New(id string, config config.HandlerConfigT) handler.Handler {
t := &SingleValueExtractorJsonpathHandler {
id: idSeq,
ready: false,
}
idSeq += 1
if config.Attributes["application"] == "" {
log.Println("Error: application not configured")
@ -77,19 +73,12 @@ func NewSvejHandler(config config.HandlerConfigT) handler.Handler {
t.unitJsonpath = jp
}
t.Id = id
t.ready = true
t.dbh = database.NewDatabaseHandle()
return t
}
func (self *SingleValueExtractorJsonpathHandler) GetId() string {
return fmt.Sprintf("SVE%d", self.id)
}
func lost(msg string, message handler.MessageT) {
log.Printf("Error: %s, message %s is lost", msg, message)
}
func extractionHelper(subTopics []string, jPayload interface{}, selector string, jp *jsonpath.Compiled) (string, error) {
var res string
switch selector[:2] {
@ -119,10 +108,10 @@ func extractionHelper(subTopics []string, jPayload interface{}, selector string,
func (self *SingleValueExtractorJsonpathHandler) Handle(message handler.MessageT) {
if ! self.ready {
log.Println("Handler is not marked as ready, message %s is lost", message)
self.Lost("Handler is not marked as ready", nil, message)
return
}
log.Printf("Handler SingleValueExtractorJsonpath %d processing %s -> %s", self.id, message.Topic, message.Payload)
//log.Printf("Handler SingleValueExtractorJsonpath %d processing %s -> %s", self.Id, message.Topic, message.Payload)
var measurement database.Measurement
measurement.Time = time.Now()
@ -133,23 +122,23 @@ func (self *SingleValueExtractorJsonpathHandler) Handle(message handler.MessageT
var jPayload interface{}
err := json.Unmarshal([]byte(message.Payload), &jPayload)
if err != nil {
lost(fmt.Sprintf("Unable to unmarshal payload: %s", err), message)
self.Lost("Unable to unmarshal payload", err, message)
return
}
device, err1 := extractionHelper(subTopics, jPayload, self.deviceSelector, self.deviceJsonpath)
if err1 != nil {
lost(fmt.Sprintf("Device extraction failed with %s", err1), message)
self.Lost("Device extraction failed", err1, message)
return
}
value, err2 := extractionHelper(subTopics, jPayload, self.valueSelector, self.valueJsonpath)
if err2 != nil {
lost(fmt.Sprintf("Value extraction failed with %s", err2), message)
self.Lost("Value extraction failed", err2, message)
return
}
unit, err3 := extractionHelper(subTopics, jPayload, self.unitSelector, self.unitJsonpath)
if err3 != nil {
lost(fmt.Sprintf("Unit extraction failed with %s", err3), message)
self.Lost("Unit extraction failed", err3, message)
return
}
@ -163,7 +152,8 @@ func (self *SingleValueExtractorJsonpathHandler) Handle(message handler.MessageT
measurement.Values = make(map[string]database.VariableType)
measurement.Values["Value"] = variable
log.Printf("Prepared measurement item: %s", measurement)
//log.Printf("Prepared measurement item: %s", measurement)
self.dbh.StoreMeasurement(&measurement)
self.S()
}

68
src/udi/handlers/sver/sver.go
View File

@ -1,28 +1,22 @@
package sver
import (
"log"
"time"
"strconv"
"strings"
"regexp"
"fmt"
"reflect"
"encoding/json"
"github.com/oliveagle/jsonpath"
"log"
"udi/config"
"udi/handlers/handler"
"udi/database"
)
var idSeq int = 0
type SingleValueExtractorRegexHandler struct {
id int
handler.CommonHandler
ready bool
config localConfig
payloadRegex *regexp.Regexp
payloadJsonpath *jsonpath.Compiled
dbh *database.DatabaseHandle
}
@ -44,12 +38,10 @@ type localConfig struct {
}
func NewSverHandler(config config.HandlerConfigT) handler.Handler {
func New(id string, config config.HandlerConfigT) handler.Handler {
t := &SingleValueExtractorRegexHandler {
id: idSeq,
ready: false,
}
idSeq += 1
var localConfig localConfig
if config.Attributes["application"] == "" {
@ -64,18 +56,6 @@ func NewSverHandler(config config.HandlerConfigT) handler.Handler {
} else {
t.payloadRegex = nil
}
payloadJsonpath := config.Attributes["payloadJsonpath"]
if payloadJsonpath != "" {
j, err := jsonpath.Compile(payloadJsonpath)
if err != nil {
log.Printf("Unable to compile jsonpath %s", payloadJsonpath)
t.payloadJsonpath = nil
} else {
t.payloadJsonpath = j
}
} else {
t.payloadJsonpath = nil
}
if config.Attributes["deviceFrom"] != TOPIC_SEL && config.Attributes["deviceFrom"] != PAYLOAD_SEL && config.Attributes["deviceFrom"] != CONSTANT_SEL {
log.Printf("Error: invalid value %s for deviceFrom", config.Attributes["deviceFrom"])
@ -128,22 +108,15 @@ func NewSverHandler(config config.HandlerConfigT) handler.Handler {
t.config = localConfig
t.Id = id
t.ready = true
t.dbh = database.NewDatabaseHandle()
return t
}
func (self *SingleValueExtractorRegexHandler) GetId() string {
return fmt.Sprintf("SVE%d", self.id)
}
func lost(msg string, message handler.MessageT) {
log.Printf("Error: %s, message %s is lost", msg, message)
}
func (self *SingleValueExtractorRegexHandler) Handle(message handler.MessageT) {
if ! self.ready {
log.Println("Handler is not marked as ready, message %s is lost", message)
self.Lost("Handler is not marked as ready", nil, message)
return
}
//log.Printf("Handler SingleValueExtractor %d processing %s -> %s", self.id, message.Topic, message.Payload)
@ -160,31 +133,21 @@ func (self *SingleValueExtractorRegexHandler) Handle(message handler.MessageT) {
payloadMatches = self.payloadRegex.FindStringSubmatch(message.Payload)
//log.Printf("Matches: %s", strings.Join(payloadMatches, ", "))
}
if self.payloadJsonpath != nil {
var jsonData interface{}
json.Unmarshal([]byte(message.Payload), &jsonData)
p, err := self.payloadJsonpath.Lookup(jsonData)
if err != nil {
lost(fmt.Sprintf("jsonpath error: %s", err), message)
return
}
log.Printf("XXXX: %s", reflect.TypeOf(p))
}
switch self.config.deviceFrom {
case TOPIC_SEL:
if self.config.devicePart >= len(subTopics) {
lost("devicePart out of range", message)
self.Lost("devicePart out of range", nil, message)
return
}
measurement.Device = subTopics[self.config.devicePart]
case PAYLOAD_SEL:
if self.payloadRegex == nil && self.payloadJsonpath == nil {
lost("no payloadRegex or payloadJsonpath defined, devicePart can't be used", message)
if self.payloadRegex == nil {
self.Lost("no payloadRegex defined, devicePart can't be used", nil, message)
return
}
if self.config.devicePart >= len(payloadMatches) {
lost("devicePart out of range", message)
self.Lost("devicePart out of range", nil, message)
return
}
measurement.Device = payloadMatches[self.config.devicePart]
@ -199,12 +162,12 @@ func (self *SingleValueExtractorRegexHandler) Handle(message handler.MessageT) {
switch self.config.valueFrom {
case PAYLOAD_SEL:
if self.payloadRegex == nil && self.payloadJsonpath == nil {
lost("no payloadRegex or payloadJsonpath defined, valuePart can't be used", message)
if self.payloadRegex == nil {
self.Lost("no payloadRegex defined, valuePart can't be used", nil, message)
return
}
if self.config.valuePart >= len(payloadMatches) {
lost("valuePart out of range", message)
self.Lost("valuePart out of range", nil, message)
return
}
variable.Value = payloadMatches[self.config.valuePart]
@ -214,12 +177,12 @@ func (self *SingleValueExtractorRegexHandler) Handle(message handler.MessageT) {
switch self.config.unitFrom {
case PAYLOAD_SEL:
if self.payloadRegex == nil && self.payloadJsonpath == nil {
lost("no payloadRegex or payloadJsonpath defined, unitPart can't be used", message)
if self.payloadRegex == nil {
self.Lost("no payloadRegex defined, unitPart can't be used", nil, message)
return
}
if self.config.unitPart >= len(payloadMatches) {
lost("unitPart out of range", message)
self.Lost("unitPart out of range", nil, message)
return
}
variable.Unit = payloadMatches[self.config.unitPart]
@ -231,5 +194,6 @@ func (self *SingleValueExtractorRegexHandler) Handle(message handler.MessageT) {
//log.Printf("Prepared measurement item: %s", measurement)
self.dbh.StoreMeasurement(&measurement)
self.S()
}

30
src/udi/handlers/ttn/ttn.go
View File

@ -1,7 +1,6 @@
package ttn
import (
"log"
"fmt"
"time"
"encoding/json"
@ -13,10 +12,9 @@ import (
"udi/database"
)
var idSeq int = 0
type TTNHandler struct {
id int
handler.CommonHandler
dbh *database.DatabaseHandle
}
@ -76,23 +74,14 @@ func (self *DecodedPayloaderHolder) UnmarshalJSON(data []byte) error {
return nil
}
func NewTTNHandler(config config.HandlerConfigT) handler.Handler {
func New(id string, config config.HandlerConfigT) handler.Handler {
t := &TTNHandler {
id: idSeq,
}
idSeq += 1
t.Id = id
t.dbh = database.NewDatabaseHandle()
return t
}
func (self *TTNHandler) GetId() string {
return fmt.Sprintf("TTN%d", self.id)
}
func lost(msg string, message handler.MessageT) {
log.Printf("Error: %s, message %s is lost", msg, message)
}
func (self *TTNHandler) Handle(message handler.MessageT) {
// log.Printf("Handler TTN %d processing %s -> %s", self.id, message.Topic, message.Payload)
@ -102,7 +91,7 @@ func (self *TTNHandler) Handle(message handler.MessageT) {
var uplinkMessage uplinkMessage
err := json.Unmarshal([]byte(message.Payload), &uplinkMessage)
if err != nil {
lost(fmt.Sprintf("Error when unmarshaling message: %s, ", err), message)
self.Lost("Error when unmarshaling message", err, message)
return
}
//log.Printf("Parsed message: %s", uplinkMessage)
@ -115,7 +104,7 @@ func (self *TTNHandler) Handle(message handler.MessageT) {
attributes.FrmPayload = uplinkMessage.UplinkMessage.FrmPayload
attributes.ConsumedAirtime = uplinkMessage.UplinkMessage.ConsumedAirtime
for _, rxm := range uplinkMessage.UplinkMessage.RxMetadata {
log.Printf("RXM: %s", rxm)
//log.Printf("RXM: %s", rxm)
g := gatewayAttributes { GatewayId: rxm.GatewayIds.GatewayId, Rssi: rxm.Rssi, Snr: rxm.Snr }
attributes.Gateways = append(attributes.Gateways, g)
}
@ -133,7 +122,7 @@ func (self *TTNHandler) Handle(message handler.MessageT) {
//log.Printf("ApplicationId: %s, DeviceId: %s", attributes.ApplicationId, attributes.DeviceId)
device, err2 := self.dbh.GetDeviceByLabelAndApplication(attributes.ApplicationId, attributes.DeviceId)
if err2 != nil {
lost(fmt.Sprintf("Error when loading device: %s, ", err2), message)
self.Lost("Error when loading device", err2, message)
return
}
measurement.Application = attributes.ApplicationId
@ -151,18 +140,19 @@ func (self *TTNHandler) Handle(message handler.MessageT) {
case "dragino-lmds200":
parser = draginoLmds200.Parse
default:
lost(fmt.Sprintf("No parser found for %s", device.DeviceType.ModelIdentifier), message)
self.Lost(fmt.Sprintf("No parser found for %s", device.DeviceType.ModelIdentifier), nil, message)
return
}
measurement.Values = make(map[string]database.VariableType)
err3 := parser(uplinkMessage.UplinkMessage.FPort, uplinkMessage.UplinkMessage.DecodedPayload.Payload, &(measurement.Values), device)
if err3 != nil {
lost(fmt.Sprintf("Model parser failed: %s", err3), message)
self.Lost("Model parser failed", err3, message)
return
}
log.Printf("Prepared measurement item: %s", measurement)
//log.Printf("Prepared measurement item: %s", measurement)
self.dbh.StoreMeasurement(&measurement)
self.S()
}