jsonpath

.gitignore

@@ -3,3 +3,4 @@ src/udi/udi
 src/udi/migrate_schema
 tmp/
 ENVDB
+ENVDB.cluster

deployment/deploy-yml.tmpl

@@ -18,6 +18,8 @@ metadata:
   namespace: udi
   labels:
     app: udi
+  annotations:
+    secret.reloader.stakater.com/reload: "udi-conf,udi-db-cred"
 spec:
   replicas: 1
   selector:
@@ -37,6 +39,9 @@ spec:
                 secretKeyRef:
                   name: udi-conf
                   key: UDI_CONF
+          envFrom:
+            - secretRef: 
+                name: udi-db-cred
           volumeMounts:
             - mountPath: /archive
               name: udi-archive

deployment/load-config.sh

@@ -23,3 +23,25 @@ kubectl create secret generic $SECRET_NAME \
   -o yaml \
   --save-config | \
 kubectl apply -f -
+
+. ~/Workspace/MyKubernetesEnv/ENVDB
+DATABASE=udi
+LOGIN=udi
+PASSWORD=`openssl rand -base64 24` 
+psql <<EOF
+ALTER USER $LOGIN WITH PASSWORD '$PASSWORD';
+GRANT ALL PRIVILEGES ON DATABASE $DATABASE TO $LOGIN;
+COMMIT;
+EOF
+
+kubectl create secret generic udi-db-cred \
+  --dry-run=client \
+  -o yaml \
+  --save-config \
+  --from-literal=PGUSER="$LOGIN" \
+  --from-literal=PGHOST="timescaledb.database.svc.cluster.local" \
+  --from-literal=PGPASSWORD="$PASSWORD" \
+  --from-literal=PGSSLMODE="require" \
+  --from-literal=PGDATABASE="$DATABASE" | \
+kubectl apply -f - -n $NAMESPACE
+

src/udi/config/config.go

@@ -17,10 +17,11 @@ type ConfigT struct {
     TlsEnable string `json:"tlsEnable"`
   } `json:"mqtt"`
   TopicMappings []struct {
+    Id string `json:"id"`
     Topics []string `json:"topics"`
     Handler string `json:"handler"`
+    Config HandlerConfigT `json:"config"`
   } `json:"topicMappings"`
-  Handlers map[string]HandlerConfigT `json:"handlers"`
   Archiver struct {
     Dir string `json:"dir"`
   }

src/udi/dispatcher/dispatcher.go

@@ -21,11 +21,37 @@ var archiverChannel chan handler.MessageT = make(chan handler.MessageT, 100)
 func InitDispatcher() {
   log.Printf("Initializing dispatcher")
   go archiver()
-  handlerMap["TTN"] = ttn.NewTTNHandler()
+
-  handlerMap["IoT"] = iot.NewIoTHandler()
+  for _, mapping := range config.Config.TopicMappings {
-  handlerMap["PV"] = pv.NewPvHandler(config.Config.Handlers["PV"])
+    // log.Printf("Trying to initialize %s", mapping)
-  handlerMap["MBGW3"] = mbgw3.NewMbgw3Handler(config.Config.Handlers["MBGW3"])
+
-  handlerMap["SVE"] = sve.NewSveHandler(config.Config.Handlers["SVE"])
+    var factory interface{}
+    switch mapping.Handler {
+    case "TTN":
+      factory = ttn.NewTTNHandler
+    case "IoT":
+      factory = iot.NewIoTHandler
+    case "PV":
+      factory = pv.NewPvHandler
+    case "MBGW3":
+      factory = mbgw3.NewMbgw3Handler
+    case "SVE":
+      factory = sve.NewSveHandler
+    default:
+      factory = nil
+      log.Printf("No handler %s found, ignore mapping", mapping.Handler)
+    }
+
+    fn, ok := factory.(func(config.HandlerConfigT) handler.Handler)
+    if ! ok {
+      log.Println("Typ Assertion failed")
+      break
+    }
+    handler := fn(mapping.Config)
+    handlerMap[mapping.Id] = handler
+  }
+
+  log.Printf("handlerMap: %s", handlerMap)
 }
 
 func storeMessage(filename string, item handler.MessageT) {
@@ -85,13 +111,13 @@ func handleMessage(message handler.MessageT) {
     for _, subscribedTopic := range mapping.Topics {
       // log.Printf("Testing %s in %s", message.Topic, subscribedTopic)
       if mqtt.TopicMatchesSubscription(message.Topic, subscribedTopic) {
-        log.Printf("Handle message in handler %s", mapping.Handler)
+        log.Printf("Handle message in handler %s", mapping.Id)
-        handler, exists := handlerMap[mapping.Handler]
+        handler, exists := handlerMap[mapping.Id]
         if exists {
           handler.Handle(message)
           return
         } else {
-          log.Printf("Handler %s not found, message %s is lost", mapping.Handler, message)
+          log.Printf("Handler %s not found, message %s is lost", mapping.Id, message)
         }
       }
     }

src/udi/go.mod

@@ -5,6 +5,7 @@ go 1.21.3
 require (
 	github.com/eclipse/paho.mqtt.golang v1.4.3
 	github.com/google/uuid v1.4.0
+	github.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852
 	gorm.io/driver/postgres v1.5.4
 	gorm.io/gorm v1.25.5
 )

src/udi/go.sum

@@ -17,6 +17,8 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
 github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852 h1:Yl0tPBa8QPjGmesFh1D0rDy+q1Twx6FyU7VWHi8wZbI=
+github.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852/go.mod h1:eqOVx5Vwu4gd2mmMZvVZsgIqNSaW3xxRThUJ0k/TPk4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

src/udi/handlers/handler/handler.go

@@ -9,6 +9,7 @@ type MessageT struct {
 }
 
 type Handler interface {
+  GetId() string
   Handle(MessageT)
 }
 

src/udi/handlers/iot/iot.go

@@ -1,6 +1,7 @@
 package iot
 
 import "log"
+import "fmt"
 import "udi/handlers/handler"
 
 var idSeq int = 0
@@ -9,7 +10,7 @@ type IoTHandler struct {
   id int
 }
 
-func NewIoTHandler() *IoTHandler {
+func NewIoTHandler() handler.Handler {
   t := &IoTHandler {
     id: idSeq,
   }
@@ -17,6 +18,10 @@ func NewIoTHandler() *IoTHandler {
   return t
 }
 
+func (self *IoTHandler) GetId() string {
+  return fmt.Sprintf("IoT%d", self.id)
+}
+
 func (self *IoTHandler) Handle(message handler.MessageT) {
   log.Printf("Handler IoT %d processing %s -> %s", self.id, message.Topic, message.Payload)
 }

src/udi/handlers/mbgw3/mbgw3.go

@@ -5,6 +5,7 @@ import (
   //"reflect"
   "time"
   "strconv"
+  "fmt"
   "encoding/json"
   "udi/config"
   "udi/handlers/handler"
@@ -30,7 +31,7 @@ type Observation struct {
 }
 
 
-func NewMbgw3Handler(config config.HandlerConfigT) *Mbgw3Handler {
+func NewMbgw3Handler(config config.HandlerConfigT) handler.Handler {
   t := &Mbgw3Handler {
     id: idSeq,
   }
@@ -39,6 +40,10 @@ func NewMbgw3Handler(config config.HandlerConfigT) *Mbgw3Handler {
   return t
 }
 
+func (self *Mbgw3Handler) GetId() string {
+  return fmt.Sprintf("MBGW3%d", self.id)
+}
+
 func (self *Mbgw3Handler) Handle(message handler.MessageT) {
 //  log.Printf("Handler MBGW3 %d processing %s -> %s", self.id, message.Topic, message.Payload)
 

src/udi/handlers/pv/pv.go

@@ -4,6 +4,7 @@ import (
   "log"
   "reflect"
   "time"
+  "fmt"
   "encoding/json"
   "udi/config"
   "udi/handlers/handler"
@@ -39,7 +40,7 @@ type PvValue struct {
 }
 
 
-func NewPvHandler(config config.HandlerConfigT) *PvHandler {
+func NewPvHandler(config config.HandlerConfigT) handler.Handler {
   t := &PvHandler {
     id: idSeq,
   }
@@ -48,6 +49,10 @@ func NewPvHandler(config config.HandlerConfigT) *PvHandler {
   return t
 }
 
+func (self *PvHandler) GetId() string {
+  return fmt.Sprintf("PV%d", self.id)
+}
+
 func (self *PvHandler) Handle(message handler.MessageT) {
   //log.Printf("Handler PV %d processing %s -> %s", self.id, message.Topic, message.Payload)
 

src/udi/handlers/sve/sve.go

@@ -6,6 +6,9 @@ import (
   "strconv"
   "strings"
   "regexp"
+  "fmt"
+  "encoding/json"
+  "github.com/oliveagle/jsonpath"
   "udi/config"
   "udi/handlers/handler"
   "udi/database"
@@ -18,11 +21,14 @@ type SingleValueExtractorHandler struct {
   ready bool
   config localConfig
   payloadRegex *regexp.Regexp
+  payloadJsonpath *jsonpath.Compiled
   dbh *database.DatabaseHandle
 }
 
 const TOPIC_SEL = "topic"
-const PAYLOAD_SEL = "payload"
+const PAYLOAD_REGEX_SEL = "payload-regex"
+const PAYLOAD_JSONPATH_SEL = "payload-jsonpath"
+const PAYLOAD_FULL_SEL = "payload-full"
 const CONSTANT_SEL = "constant"
 
 type localConfig struct {
@@ -38,7 +44,7 @@ type localConfig struct {
 }
 
 
-func NewSveHandler(config config.HandlerConfigT) *SingleValueExtractorHandler {
+func NewSveHandler(config config.HandlerConfigT) handler.Handler {
   t := &SingleValueExtractorHandler {
     id: idSeq,
     ready: false,
@@ -58,8 +64,20 @@ func NewSveHandler(config config.HandlerConfigT) *SingleValueExtractorHandler {
   } else {
     t.payloadRegex = nil
   }
+  payloadJsonpath := config.Attributes["payloadJsonpath"]
+  if payloadJsonpath != "" {
+    j, err := jsonpath.Compile(payloadJsonpath)
+    if err != nil {
+      log.Printf("Unable to compile jsonpath %s", payloadJsonpath)
+      t.payloadJsonpath = nil
+    } else {
+      t.payloadJsonpath = j
+    }
+  } else {
+    t.payloadJsonpath = nil
+  }
 
-  if config.Attributes["deviceFrom"] != TOPIC_SEL && config.Attributes["deviceFrom"] != PAYLOAD_SEL && config.Attributes["deviceFrom"] != CONSTANT_SEL {
+  if config.Attributes["deviceFrom"] != TOPIC_SEL && config.Attributes["deviceFrom"] != PAYLOAD_REGEX_SEL && config.Attributes["deviceFrom"] != CONSTANT_SEL {
     log.Printf("Error: invalid value %s for deviceFrom", config.Attributes["deviceFrom"])
     return t
   }
@@ -75,31 +93,35 @@ func NewSveHandler(config config.HandlerConfigT) *SingleValueExtractorHandler {
   // empty device is valid
   localConfig.device = config.Attributes["device"]
 
-  if config.Attributes["valueFrom"] != TOPIC_SEL && config.Attributes["valueFrom"] != PAYLOAD_SEL {
+  if config.Attributes["valueFrom"] != PAYLOAD_REGEX_SEL && config.Attributes["valueFrom"] != PAYLOAD_JSONPATH_SEL && config.Attributes["valueFrom"] != PAYLOAD_FULL_SEL {
     log.Printf("Error: invalid value %s for valueFrom", config.Attributes["valueFrom"])
     return t
   }
   localConfig.valueFrom = config.Attributes["valueFrom"]
 
-  valuePart, err2 := strconv.Atoi(config.Attributes["valuePart"])
+  if config.Attributes["valueFrom"] == PAYLOAD_REGEX_SEL {
-  if err2 != nil {
+    valuePart, err2 := strconv.Atoi(config.Attributes["valuePart"])
-    log.Printf("Error: unable to convert valuePart to number: %s", err2)
+    if err2 != nil {
-    return t
+      log.Printf("Error: unable to convert valuePart to number: %s", err2)
+      return t
+    }
+    localConfig.valuePart = valuePart
   }
-  localConfig.valuePart = valuePart
 
-  if config.Attributes["unitFrom"] != TOPIC_SEL && config.Attributes["unitFrom"] != PAYLOAD_SEL && config.Attributes["unitFrom"] != CONSTANT_SEL {
+  if config.Attributes["unitFrom"] != TOPIC_SEL && config.Attributes["unitFrom"] != PAYLOAD_REGEX_SEL && config.Attributes["unitFrom"] != CONSTANT_SEL {
     log.Printf("Error: invalid value %s for unitFrom", config.Attributes["unitFrom"])
     return t
   }
   localConfig.unitFrom = config.Attributes["unitFrom"]
 
-  unitPart, err3 := strconv.Atoi(config.Attributes["unitPart"])
+  if config.Attributes["unitFrom"] != CONSTANT_SEL {
-  if err3 != nil {
+    unitPart, err3 := strconv.Atoi(config.Attributes["unitPart"])
-    log.Printf("Error: unable to convert unitPart to number: %s", err3)
+    if err3 != nil {
-    return t
+      log.Printf("Error: unable to convert unitPart to number: %s", err3)
+      return t
+    }
+    localConfig.unitPart = unitPart
   }
-  localConfig.unitPart = unitPart
 
   // empty unit is valid
   localConfig.unit = config.Attributes["unit"]
@@ -111,6 +133,10 @@ func NewSveHandler(config config.HandlerConfigT) *SingleValueExtractorHandler {
   return t
 }
 
+func (self *SingleValueExtractorHandler) GetId() string {
+  return fmt.Sprintf("SVE%d", self.id)
+}
+
 func lost(msg string, message handler.MessageT) {
   log.Printf("Error: %s, message %s is lost", msg, message)
 }
@@ -142,7 +168,7 @@ func (self *SingleValueExtractorHandler) Handle(message handler.MessageT) {
       return
     }
     measurement.Device = subTopics[self.config.devicePart]
-  case PAYLOAD_SEL:
+  case PAYLOAD_REGEX_SEL:
     if self.payloadRegex == nil {
       lost("no payloadRegex defined, devicePart can't be used", message)
       return
@@ -162,22 +188,31 @@ func (self *SingleValueExtractorHandler) Handle(message handler.MessageT) {
   variable.Variable = ""
 
   switch self.config.valueFrom {
-  case TOPIC_SEL:
+  case PAYLOAD_REGEX_SEL:
-    if self.config.valuePart >= len(subTopics) {
-      lost("valuePart out of range", message)
-      return
-    }
-    variable.Value = subTopics[self.config.valuePart]
-  case PAYLOAD_SEL:
     if self.payloadRegex == nil {
       lost("no payloadRegex defined, valuePart can't be used", message)
       return
     }
-    if self.config.valuePart >= len(subTopics) {
+    if self.config.valuePart >= len(payloadMatches) {
       lost("valuePart out of range", message)
       return
     }
     variable.Value = payloadMatches[self.config.valuePart]
+  case PAYLOAD_JSONPATH_SEL:
+    if self.payloadJsonpath == nil {
+      lost("no payloadJsonpath defined, valuePart can't be used", message)
+      return
+    }
+    var jsonData interface{}
+    json.Unmarshal([]byte(message.Payload), &jsonData)
+    result, err := self.payloadJsonpath.Lookup(jsonData)
+    if err != nil {
+      lost(fmt.Sprintf("jsonpath error: %s", err), message)
+      return
+    }
+    variable.Value = result
+  case PAYLOAD_FULL_SEL:
+    variable.Value = message.Payload
   }
 
   switch self.config.unitFrom {
@@ -187,7 +222,7 @@ func (self *SingleValueExtractorHandler) Handle(message handler.MessageT) {
       return
     }
     variable.Unit = subTopics[self.config.unitPart]
-  case PAYLOAD_SEL:
+  case PAYLOAD_REGEX_SEL:
     if self.payloadRegex == nil {
       lost("no payloadRegex defined, unitPart can't be used", message)
       return

src/udi/handlers/ttn/ttn.go

@@ -1,6 +1,7 @@
 package ttn
 
 import "log"
+import "fmt"
 import "udi/handlers/handler"
 
 var idSeq int = 0
@@ -9,7 +10,7 @@ type TTNHandler struct {
   id int
 }
 
-func NewTTNHandler() *TTNHandler {
+func NewTTNHandler() handler.Handler {
   t := &TTNHandler {
     id: idSeq,
   }
@@ -17,6 +18,10 @@ func NewTTNHandler() *TTNHandler {
   return t
 }
 
+func (self *TTNHandler) GetId() string {
+  return fmt.Sprintf("TTN%d", self.id)
+}
+
 func (self *TTNHandler) Handle(message handler.MessageT) {
   log.Printf("Handler TTN %d processing %s -> %s", self.id, message.Topic, message.Payload)
 }