drop scan

.gitignore

@@ -1,4 +1,5 @@
 src/udi/udi
+src/udi/main
 src/udi/migrate_schema
 tmp/
 ENVDB

.woodpecker.yml

@@ -14,13 +14,6 @@ steps:
     when:
       - event: [push, tag]
 
-  scan_image:
-    image: aquasec/trivy
-    commands:
-      - trivy image $FORGE_NAME/$CI_REPO:$CI_COMMIT_SHA --quiet --exit-code 1
-    when:
-      - event: [push, tag]
-        
   deploy:
     image: portainer/kubectl-shell:latest
     secrets:

deployment/instances/udi/default/config.json

@@ -119,6 +119,16 @@
         }
       }
     },
+    {
+      "topics": [ "zigbee2mqtt/+" ],
+      "handler": "Z2M",
+      "id": "Z2M",
+      "config": {
+        "databaseConnStr": "",
+        "attributes": {
+        }
+      }
+    },
     {
       "topics": [ "shellyplusht/+/status/temperature:0" ],
       "handler": "SVEJ",

queries/saerbeck.sql

@@ -10,14 +10,34 @@ create or replace view badesee_temperature_v as
     where application = 'de-hottis-saerbeck-monitoring' and
           device = 'eui-a84041318187ec13';
 
-create or replace view cubecell_threeway_temperature_v as
+create or replace view cubecell_threeway_temperature2_v as
   select time,
-         cast(values->'Temperature1'->>'value' as float) as Temp1,
+         cast(values->'Temperature2'->>'value' as float) as value,
-         cast(values->'Temperature2'->>'value' as float) as Temp2,
+         values->'Temperature2'->>'label' as label
-         cast(values->'Temperature3'->>'value' as float) as Temp3,
-         device
     from measurements
     where application = 'de-hottis-saerbeck-monitoring' and
     device = 'eui-70b3d57ed0068fa4';
 
+create or replace view cubecell_threeway_temperature1_v as
+  select time,
+         cast(values->'Temperature1'->>'value' as float) as value,
+         values->'Temperature1'->>'label' as label
+    from measurements
+    where application = 'de-hottis-saerbeck-monitoring' and
+    device = 'eui-70b3d57ed0068fa4';
 
+create or replace view cubecell_threeway_temperature3_v as
+  select time,
+         cast(values->'Temperature3'->>'value' as float) as value,
+         values->'Temperature3'->>'label' as label
+    from measurements
+    where application = 'de-hottis-saerbeck-monitoring' and
+    device = 'eui-70b3d57ed0068fa4';
+
+create or replace view cubecell_threeway_battery_v as
+  select time,
+         cast(values->'Battery'->>'value' as float) as value,
+         values->'Battery'->>'label' as label
+    from measurements
+    where application = 'de-hottis-saerbeck-monitoring' and
+    device = 'eui-70b3d57ed0068fa4';

src/udi/ENVDB.uditest

@@ -1,6 +1,6 @@
-PGUSER="uditest"
+PGUSER="udi-hottis"
 PGHOST=`kubectl get services traefik -n system -o jsonpath="{.status.loadBalancer.ingress[0].ip}"`
-PGPASSWORD=`kubectl get secrets uditest-db-cred -n udi-test -o jsonpath="{.data.PGPASSWORD}" | base64 --decode`
+PGPASSWORD=`kubectl get secrets default-udi-db-cred -n udi -o jsonpath="{.data.PGPASSWORD}" | base64 --decode`
 PGSSLMODE=require
 PGDATABASE="uditest"
 export PGUSER PGHOST PGPASSWORD PGSSLMODE PGDATABASE

src/udi/config-test.json

@@ -4,72 +4,6 @@
     "tlsEnable": "false"
   },
   "topicMappings": [
-    {
-      "topics": [ "IoT/PV/Values" ],
-      "handler": "PV",
-      "id": "PV",
-      "config": {
-        "attributes": {
-        }
-      }
-    },
-    {
-      "topics": [ "IoT/MBGW3/Measurement" ],
-      "handler": "MBGW3",
-      "id": "MBGW3",
-      "config": {
-        "attributes": {
-        }
-      }
-    },
-    {
-      "topics": [ "dt1/ai/periodic/1" ],
-      "handler": "DT1T",
-      "id": "DT1T.0",
-      "config": {
-        "attributes": {
-          "Application": "Temperature Wago",
-          "Device": "Freezer",
-          "HardLow": "-273",
-          "SoftLow": "-50",
-          "SoftHigh": "20",
-          "HardHigh": "100"
-        }
-      }
-    },
-    {
-      "topics": [ "dt1/ai/periodic/3" ],
-      "handler": "DT1T",
-      "id": "DT1T.1",
-      "config": {
-        "attributes": {
-          "Application": "Temperature Wago",
-          "Device": "Outdoor",
-          "HardLow": "-273",
-          "SoftLow": "-60",
-          "SoftHigh": "60",
-          "HardHigh": "100"
-        }
-      }
-    },
-    {
-      "topics": [ "IoT/OneWireGW/Bus 1/#" ],
-      "handler": "SVER",
-      "id": "SVER0",
-      "config": {
-        "databaseConnStr": "",
-        "attributes": {
-          "application": "Temperature Heating",
-          "payloadRegex": "(\\d+(\\.\\d+)?)\\s*([^0-9\\s]\\S*)",
-          "deviceFrom": "topic", 
-          "devicePart": "3",
-          "valueFrom": "payload",
-          "valuePart": "1",
-          "unitFrom": "payload",
-          "unitPart": "3"
-        }
-      }
-    },
     {
       "topics": [ "NR/Multisensor/+/Temperatur" ],
       "handler": "SVEJ",
@@ -85,44 +19,12 @@
       }
     },
     {
-      "topics": [ "NR/Multisensor/+/Feuchte" ],
+      "topics": [ "zigbee2mqtt/+" ],
-      "handler": "SVEJ",
+      "handler": "Z2M",
-      "id": "SVEJ1",
+      "id": "Z2M",
       "config": {
         "databaseConnStr": "",
         "attributes": {
-          "application": "Humidity Multisensor",
-          "deviceSelector": "T:2",
-          "valueSelector": "J:$.CurrentRelativeHumidity",
-          "unitSelector": "C:%"
-        }
-      }
-    },
-    {
-      "topics": [ "shellyplusht/+/status/temperature:0" ],
-      "handler": "SVEJ",
-      "id": "SVEJ2",
-      "config": {
-        "databaseConnStr": "",
-        "attributes": {
-          "application": "Temperature Shelly Plus HT",
-          "deviceSelector": "T:1",
-          "valueSelector": "J:$.tC",
-          "unitSelector": "C:°C"
-        }
-      }
-    },
-    {
-      "topics": [ "shellyplusht/+/status/humidity:0" ],
-      "handler": "SVEJ",
-      "id": "SVE4",
-      "config": {
-        "databaseConnStr": "",
-        "attributes": {
-          "application": "Humidity Shelly Plus HT",
-          "deviceSelector": "T:1",
-          "valueSelector": "J:$.rh",
-          "unitSelector": "C:%"
         }
       }
     }

src/udi/database/database.go

@@ -70,5 +70,27 @@ func (self *DatabaseHandle) GetDeviceByLabelAndApplication(applicationLabel stri
   return &device, nil
 }
 
+func (self *DatabaseHandle) GetDeviceByLabel(deviceLabel string) (*Device, error) {
+  if ! self.initialized {
+    err := fmt.Errorf("Database connection not initialized")
+    return nil, err
+  }
+
+  var device Device
+  result := self.dbh.
+    Preload("Application").
+    Preload("DeviceType").
+    Where("devices.label = ?", deviceLabel).
+    First(&device)
+
+  if result.Error != nil {
+    err := fmt.Errorf("Query failed: %s", result.Error)
+    return nil, err
+  }
+
+  return &device, nil
+}
+
+
 
 

src/udi/dispatcher/dispatcher.go

@@ -18,6 +18,7 @@ import "udi/handlers/svej"
 import "udi/handlers/dt1t"
 import "udi/handlers/locative"
 import "udi/handlers/snmp"
+import "udi/handlers/z2m"
 
 
 var handlerMap map[string]handler.Handler = make(map[string]handler.Handler)
@@ -50,6 +51,8 @@ func InitDispatcher() {
       factory = locative.New
     case "SNMP":
       factory = snmp.New
+    case "Z2M":
+      factory = z2m.New
     default:
       factory = nil
       log.Printf("No handler %s found, ignore mapping", mapping.Handler)

src/udi/handlers/svej/svej.go

@@ -79,16 +79,18 @@ func New(id string, config config.HandlerConfigT) handler.Handler {
   return t
 }
 
-func extractionHelper(subTopics []string, jPayload interface{}, selector string, jp *jsonpath.Compiled) (string, error) {
+func (self *SingleValueExtractorJsonpathHandler) ExtractionHelper(subTopics []string, jPayload interface{}, selector string, jp *jsonpath.Compiled) (string, error) {
   var res string
   switch selector[:2] {
   case "J:":
+    // extract using jsonpath from payload
     r, e := jp.Lookup(jPayload)
     if e != nil {
       return "", fmt.Errorf("jp.Lookup failed with %s", e)
     }
     res = fmt.Sprint(r)
   case "T:":
+    // T: extract from topic
     i, e := strconv.Atoi(selector[2:])
     if e != nil {
       return "", fmt.Errorf("Atoi failed with %s", e)
@@ -98,6 +100,7 @@ func extractionHelper(subTopics []string, jPayload interface{}, selector string,
     }
     res = subTopics[i]
   case "C:":
+    // use constant value
     res = selector[2:]
   default:
     return "", fmt.Errorf("Invalid selector: %s", selector[:2])
@@ -111,14 +114,14 @@ func (self *SingleValueExtractorJsonpathHandler) Handle(message handler.MessageT
     self.Lost("Handler is not marked as ready", nil, message)
     return
   }
-  //log.Printf("Handler SingleValueExtractorJsonpath %d processing %s -> %s", self.Id, message.Topic, message.Payload)
+  log.Printf("Handler SingleValueExtractorJsonpath %d processing %s -> %s", self.Id, message.Topic, message.Payload)
 
   var measurement database.Measurement
   measurement.Time = time.Now()
   measurement.Application = self.application
   
   subTopics := strings.Split(message.Topic, "/")
-  //log.Printf("Subtopics: %s", strings.Join(subTopics, ", "))
+  log.Printf("Subtopics: %s", strings.Join(subTopics, ", "))
   var jPayload interface{}
   err := json.Unmarshal([]byte(message.Payload), &jPayload)
   if err != nil {
@@ -126,17 +129,20 @@ func (self *SingleValueExtractorJsonpathHandler) Handle(message handler.MessageT
     return
   }
 
-  device, err1 := extractionHelper(subTopics, jPayload, self.deviceSelector, self.deviceJsonpath)
+  device, err1 := self.ExtractionHelper(subTopics, jPayload, self.deviceSelector, self.deviceJsonpath)
   if err1 != nil {
     self.Lost("Device extraction failed", err1, message)
     return
   }
-  value, err2 := extractionHelper(subTopics, jPayload, self.valueSelector, self.valueJsonpath)
+  log.Printf("device: %s", device) 
+
+  value, err2 := self.ExtractionHelper(subTopics, jPayload, self.valueSelector, self.valueJsonpath)
   if err2 != nil {
     self.Lost("Value extraction failed", err2, message)
     return
   }
-  unit, err3 := extractionHelper(subTopics, jPayload, self.unitSelector, self.unitJsonpath)
+
+  unit, err3 := self.ExtractionHelper(subTopics, jPayload, self.unitSelector, self.unitJsonpath)
   if err3 != nil {
     self.Lost("Unit extraction failed", err3, message)
     return
@@ -152,7 +158,7 @@ func (self *SingleValueExtractorJsonpathHandler) Handle(message handler.MessageT
   measurement.Values = make(map[string]database.VariableType)
   measurement.Values["Value"] = variable
 
-  //log.Printf("Prepared measurement item: %s", measurement)
+  log.Printf("Prepared measurement item: %s", measurement)
   self.dbh.StoreMeasurement(&measurement)
   self.S()
 }

src/udi/handlers/ttn/models/hottisThreeWayThermometer/hottisThreeWayThermometer.go

@@ -7,7 +7,6 @@ import (
   "strconv"
   "encoding/base64"
   "encoding/binary"
-  "encoding/json"
   "udi/database"
 )
 
@@ -22,24 +21,17 @@ type hottisThreeWayThermometerValues struct {
   Value3 int32 
 }
 
-func getSensorName(sensorsMap *map[string]string, sensorAddress uint64) string {
+func getSensorName(sensorsMap *map[string]interface{}, sensorAddress uint64) string {
   key := strconv.FormatUint(sensorAddress, 10)
-	if sensorName, exists := (*sensorsMap)[key]; exists {
+	if sensorName, exists := (*sensorsMap)[key].(string); exists {
 		return sensorName
 	}
 	return "Sensor" + key
 }
 
 func Parse(fPort int, _ []byte, frmPayload string, variables *map[string]database.VariableType, attributes *map[string]interface{}, device *database.Device) error {
-  sensorsMap := make(map[string]string)
+  deviceAttrs := (*device).Attributes
-  sensorsJSON, ok := (*device).Attributes["Sensors"].(string)
+  sensorsMap := deviceAttrs["Sensors"].(map[string]interface{})
-  if !ok {
-    return fmt.Errorf("Unable to load sensor map from attributes")
-  }
-  errJ := json.Unmarshal([]byte(sensorsJSON), &sensorsMap)
-  if errJ != nil {
-    return fmt.Errorf("Unable to parse sensor map: %v", errJ)
-  }
 
   if fPort != 2 {
     return fmt.Errorf("Unexpected fPort %d", fPort)

src/udi/handlers/z2m/models/wsdcgq11lm/wsdcgq11lm.go

@@ -0,0 +1,11 @@
+package wsdcgq11lm
+
+type Observation struct {
+  LinkQuality uint8 `unit:"" json:"linkquality"`
+  Battery uint8 `unit:"%" json:"battery"`
+  Humidity float32 `unit:"%H" json:"humidity"`
+  Pressure float32 `unit:"mbar" json:"pressure"`
+  Temperature float32 `unit:"°C" json:"temperature"`
+  Voltage uint16 `unit:"mV" json:"voltage"`
+}
+

src/udi/handlers/z2m/z2m.go

@@ -0,0 +1,106 @@
+package z2m
+
+import (
+  "fmt"
+  "log"
+  "time"
+  "strings"
+  "reflect"
+  "encoding/json"
+  "udi/config"
+  "udi/handlers/handler"
+  "udi/database"
+  "udi/handlers/z2m/models/wsdcgq11lm"
+)
+
+
+type Z2MHandler struct {
+  handler.CommonHandler
+  dbh *database.DatabaseHandle
+}
+
+func parse(T any, payload string, variables *map[string]database.VariableType) error {
+  observationType := reflect.TypeOf(T)
+  observation := reflect.New(observationType).Interface()
+
+  err := json.Unmarshal([]byte(payload), observation)
+  if err != nil {
+    return fmt.Errorf("Unable to parse payload into Observation struct: %v, %s", err, payload)
+  }
+
+  observationValue := reflect.ValueOf(observation).Elem()
+
+  for i := 0; i < observationType.NumField(); i++ {
+    field := observationType.Field(i)
+    name := field.Name
+    unit := field.Tag.Get("unit")
+    value := observationValue.Field(i).Interface()
+
+    (*variables)[name] = database.VariableType {
+      Label: name,
+      Variable: "y",
+      Unit: unit,
+      Value: value,
+    }
+  }
+
+
+  return nil
+}
+
+func New(id string, config config.HandlerConfigT) handler.Handler {
+  t := &Z2MHandler {
+  }
+  t.Id = id
+  t.dbh = database.NewDatabaseHandle()
+  return t
+}
+
+func (self *Z2MHandler) Handle(message handler.MessageT) {
+  log.Printf("Handler Z2M %d processing %s -> %s", self.Id, message.Topic, message.Payload)
+
+  var measurement database.Measurement
+  measurement.Time = time.Now()
+
+
+  subTopics := strings.Split(message.Topic, "/")
+  deviceId := subTopics[1]
+  log.Printf("DeviceId: %s", deviceId)
+  device, err1 := self.dbh.GetDeviceByLabel(deviceId)
+  if err1 != nil {
+    self.Lost("Error when loading device", err1, message)
+    return
+  }
+  log.Printf("Device: %s", device)
+
+  measurement.Application = device.Application.Label
+  measurement.Device = device.Attributes["Label"].(string)
+
+  var T any
+  switch device.DeviceType.ModelIdentifier {
+  case "WSDCGQ11LM":
+    T = wsdcgq11lm.Observation{}
+  default:
+    self.Lost(fmt.Sprintf("No parser found for %s", device.DeviceType.ModelIdentifier), nil, message)
+    return
+  }
+
+  measurement.Values = make(map[string]database.VariableType)
+  measurement.Attributes = make(map[string]interface{})
+  err3 := parse(T, 
+                message.Payload,
+                &(measurement.Values))
+  if err3 != nil {
+    self.Lost("Model parser failed", err3, message)
+    return
+  }
+
+  measurement.Attributes["Status"] = "ok"
+  measurement.Attributes["DeviceId"] = deviceId
+  measurement.Attributes["DeviceModel"] = device.DeviceType.ModelIdentifier
+  log.Printf("Prepared measurement item: %s", measurement)
+  self.dbh.StoreMeasurement(&measurement)
+  self.S()
+}
+
+