new database

.gitignore

@@ -1,4 +1,5 @@
 src/udi/udi
+src/udi/main
 src/udi/migrate_schema
 tmp/
 ENVDB

.woodpecker.yml

@@ -14,13 +14,6 @@ steps:
     when:
       - event: [push, tag]
 
-  scan_image:
-    image: aquasec/trivy
-    commands:
-      - trivy image $FORGE_NAME/$CI_REPO:$CI_COMMIT_SHA --quiet --exit-code 1
-    when:
-      - event: [push, tag]
-        
   deploy:
     image: portainer/kubectl-shell:latest
     secrets:

deployment/decrypt-secrets.sh

@@ -10,6 +10,7 @@ if [ "$MD5_CHECKSUM" = "" ]; then
   exit 1
 fi
 
+
 SECRETS_CIPHERTEXT_FILE=secrets.enc
 SECRETS_PLAINTEXT_FILE=/tmp/secrets
 TMP_FILE=`mktemp`
@@ -37,7 +38,7 @@ if [ "$MD5_CHECKSUM" != "$CALCULATED_CHECKSUM" ]; then
   exit 1
 fi
 
-# cat $TMP_FILE
+#cat $TMP_FILE
 mv $TMP_FILE $SECRETS_PLAINTEXT_FILE
 
 

deployment/deploy.sh

@@ -58,7 +58,7 @@ for NAMESPACE_DIR in `find $INSTANCES_DIR -type d -mindepth 1 -maxdepth 1`; do
     NEW_UDI_DB_PASSWORD="${!PASSWORD_VARIABLE}"
     DATABASE_VARIABLE=$VARIABLE_PREFIX"_PGDATABASE"
     NEW_UDI_DB_DATABASE="${!DATABASE_VARIABLE}"
-    NEW_UDI_DB_HOST=timescaledb.database.svc.cluster.local
+    NEW_UDI_DB_HOST=database.database1.svc.cluster.local
 
     kubectl create secret generic $INSTANCE-udi-db-cred \
       --dry-run=client \

deployment/instances/udi/default/config.json

@@ -119,6 +119,16 @@
         }
       }
     },
+    {
+      "topics": [ "zigbee2mqtt/+" ],
+      "handler": "Z2M",
+      "id": "Z2M",
+      "config": {
+        "databaseConnStr": "",
+        "attributes": {
+        }
+      }
+    },
     {
       "topics": [ "shellyplusht/+/status/temperature:0" ],
       "handler": "SVEJ",

queries/hottis.sql

@@ -43,7 +43,22 @@ create or replace view temperature_v as
          cast(values->'Value'->>'value' as float) as temperature,
          device
     from measurements
-    where application in ('Temperature Multisensor', 'Temperature Shelly Plus HT');
+    where application in ('Temperature Multisensor', 'Temperature Shelly Plus HT')
+  union
+  select time,
+         cast(values->'Temperature'->>'value' as float) as temperature,
+         device
+    from measurements
+    where application = 'Zigbee2MQTT Hottis Eupenstr.' and
+          attributes->>'DeviceModel' in ('WSDCGQ11LM', 'WSDCGQ01LM');
+
+create or replace view voltage_v as
+  select time,
+         cast(values->'Voltage'->>'value' as float) / 1000 as voltage,
+         device
+    from measurements
+    where application = 'Zigbee2MQTT Hottis Eupenstr.' and
+          attributes->>'DeviceModel' in ('WSDCGQ11LM', 'WSDCGQ01LM');
 
 create or replace view temperature2_v as
   select time,
@@ -57,7 +72,14 @@ create or replace view humidity_v as
          cast(values->'Value'->>'value' as float) as humidity,
          device
     from measurements
-    where application in ('Humidity Multisensor');
+    where application in ('Humidity Multisensor')
+  union
+  select time,
+         cast(values->'Humidity'->>'value' as float) as temperature,
+         device
+    from measurements
+    where application = 'Zigbee2MQTT Hottis Eupenstr.' and
+          attributes->>'DeviceModel' in ('WSDCGQ11LM', 'WSDCGQ01LM');
 
 create or replace view soil_v as
   select time,
@@ -96,3 +118,14 @@ create or replace view router_v as
          cast(values->'wan-out'->>'value'  as int) as wanOutOctetsPerSeconds
     from measurements
     where application = 'SNMP' and device = '172.16.3.1';
+
+create or replace view lora_sht21_v as
+  select time,
+         cast(values->'Humidity'->>'value' as float) as humidity,
+         cast(values->'Temperature'->>'value' as float) as temperature,
+         m.device as device,
+         d.attributes->>'Label' as label
+    from measurements m, devices d
+    where m.application = 'de-hottis-app01' and 
+          m.attributes->>'DeviceType' = 'hottis-gy21' and
+          m.device = d.label;

src/udi/config-test.json

@@ -20,15 +20,11 @@
     },
     {
       "topics": [ "zigbee2mqtt/+" ],
-      "handler": "SVEJ",
-      "id": "SVEJ1",
+      "handler": "Z2M",
+      "id": "Z2M",
       "config": {
         "databaseConnStr": "",
         "attributes": {
-          "application": "Temperature Multisensor",
-          "deviceSelector": "L:1",
-          "valueSelector": "J:$.temperature",
-          "unitSelector": "C:°C"
         }
       }
     }

src/udi/database/database.go

@@ -25,7 +25,7 @@ func NewDatabaseHandle() *DatabaseHandle {
   } else {
     db.dbh = conn
     db.initialized = true
-    //log.Println("Database connection opened")
+    log.Println("Database connection opened")
   }
   return &db
 }
@@ -44,7 +44,7 @@ func (self *DatabaseHandle) StoreMeasurement(measurement *Measurement) {
     return
   }
 
-  //log.Println("Successfully stored measurement")
+  log.Println("Successfully stored measurement")
   counter.S("Stored")
 }
 

src/udi/dispatcher/dispatcher.go

@@ -18,6 +18,7 @@ import "udi/handlers/svej"
 import "udi/handlers/dt1t"
 import "udi/handlers/locative"
 import "udi/handlers/snmp"
+import "udi/handlers/z2m"
 
 
 var handlerMap map[string]handler.Handler = make(map[string]handler.Handler)
@@ -50,6 +51,8 @@ func InitDispatcher() {
       factory = locative.New
     case "SNMP":
       factory = snmp.New
+    case "Z2M":
+      factory = z2m.New
     default:
       factory = nil
       log.Printf("No handler %s found, ignore mapping", mapping.Handler)

src/udi/handlers/dt1t/dt1t.go

@@ -38,6 +38,7 @@ func New(id string, config config.HandlerConfigT) handler.Handler {
   t.Id = id
   
   t.dbh = database.NewDatabaseHandle()
+  log.Printf("Handler DT1T %d initialized", id)
   t.ready = true
   return t
 }

src/udi/handlers/locative/locative.go

@@ -31,6 +31,7 @@ func New(id string, config config.HandlerConfigT) handler.Handler {
   }
   t.Id = id
   t.dbh = database.NewDatabaseHandle()
+  log.Printf("Handler Locative %d initialized", id)
   return t
 }
 

src/udi/handlers/mbgw3/mbgw3.go

@@ -2,6 +2,7 @@ package mbgw3
 
 import (
   "time"
+  "log"
   "strconv"
   "encoding/json"
   "udi/config"
@@ -31,6 +32,7 @@ func New(id string, config config.HandlerConfigT) handler.Handler {
   }
   t.Id = id
   t.dbh = database.NewDatabaseHandle()
+  log.Printf("Handler MBGW3 %d initialized", id)
   return t
 }
 

src/udi/handlers/pv/pv.go

@@ -3,6 +3,7 @@ package pv
 import (
   "reflect"
   "time"
+  "log"
   "encoding/json"
   "udi/config"
   "udi/handlers/handler"
@@ -41,6 +42,7 @@ func New(id string, config config.HandlerConfigT) handler.Handler {
   }
   t.Id = id
   t.dbh = database.NewDatabaseHandle()
+  log.Printf("Handler PV %d initialized", id)
   return t
 }
 

src/udi/handlers/snmp/snmp.go

@@ -33,11 +33,12 @@ func New(id string, config config.HandlerConfigT) handler.Handler {
   }
   t.Id = id
   t.dbh = database.NewDatabaseHandle()
+  log.Printf("Handler SNMP %d initialized", id)
   return t
 }
 
 func (self *SnmpHandler) Handle(message handler.MessageT) {
-  log.Printf("Handler SNMP %d processing %s -> %s", self.Id, message.Topic, message.Payload)
+  // log.Printf("Handler SNMP %d processing %s -> %s", self.Id, message.Topic, message.Payload)
 
   var observation observation_t
   err := json.Unmarshal([]byte(message.Payload), &observation)
@@ -66,7 +67,7 @@ func (self *SnmpHandler) Handle(message handler.MessageT) {
     }
   }
 
-  log.Printf("Prepared measurement item: %s", measurement)
+  // log.Printf("Prepared measurement item: %s", measurement)
 
   self.dbh.StoreMeasurement(&measurement)
   self.S()

src/udi/handlers/svej/svej.go

@@ -76,6 +76,7 @@ func New(id string, config config.HandlerConfigT) handler.Handler {
   t.Id = id
   t.ready = true
   t.dbh = database.NewDatabaseHandle()
+  log.Printf("Handler SVEJ %d initialized", id)
   return t
 }
 
@@ -99,24 +100,6 @@ func (self *SingleValueExtractorJsonpathHandler) ExtractionHelper(subTopics []st
       return "", fmt.Errorf("not enough subtopics")
     }
     res = subTopics[i]
-  case "L:":
-    // L: extract from topic and later match against devices table in database
-    i, e := strconv.Atoi(selector[2:])
-    if e != nil {
-      return "", fmt.Errorf("Atoi failed with %s", e)
-    }
-    if i >= len(subTopics) {
-      return "", fmt.Errorf("not enough subtopics")
-    }
-    ext := subTopics[i]
-    lookup, err1b := self.dbh.GetDeviceByLabel(ext)
-    if err1b != nil {
-      log.Printf("ext lookup %s failed: %v", ext, err1b)
-      res = ext
-    } else {
-      log.Printf("ext: %s", lookup)
-      res = ext
-    }
   case "C:":
     // use constant value
     res = selector[2:]

src/udi/handlers/sver/sver.go

@@ -111,6 +111,7 @@ func New(id string, config config.HandlerConfigT) handler.Handler {
   t.Id = id
   t.ready = true
   t.dbh = database.NewDatabaseHandle()
+  log.Printf("Handler SVER %d initialized", id)
   return t
 }
 

src/udi/handlers/ttn/models/hottisGy21/hottisGY21.go

@@ -0,0 +1,55 @@
+package hottisGy21
+
+import (
+  //"log"
+  "fmt"
+  "bytes"
+  "encoding/base64"
+  "encoding/binary"
+  "udi/database"
+)
+
+type hottisGy21Values struct {
+  Connected uint8
+  Status uint8
+  RawHumidity uint16
+  RawTemperature uint16
+}
+
+
+func Parse(fPort int, _ []byte, frmPayload string, variables *map[string]database.VariableType, attributes *map[string]interface{}, _ *database.Device) error {
+  if fPort != 2 {
+    return fmt.Errorf("Unexpected fPort %d", fPort)
+  }
+
+  b, err := base64.StdEncoding.DecodeString(frmPayload)
+  if err != nil {
+    return fmt.Errorf("Unable to base64-decode payload: %v", err)
+  }
+
+  var values hottisGy21Values
+  err = binary.Read(bytes.NewReader(b), binary.LittleEndian, &values)
+  if err != nil {
+    return fmt.Errorf("Unable to cast into struct: %v", err)
+  }
+  var temperature float32 = -46.85 + 175.72 * (float32(values.RawTemperature) / 65536.0)
+  var humidity float32 = -6 + 125 * (float32(values.RawHumidity) / 65536.0);
+
+  // log.Printf("CO2: %f, Temp: %f, Hum: %f, Status: %d", co2concentration, temperature, humidity, values.Status)
+
+  (*variables)["Humidity"] = database.VariableType {
+    Label: "Humidity",
+    Variable: "Humidity",
+    Unit: "%",
+    Value: humidity,
+  }
+  (*variables)["Temperature"] = database.VariableType {
+    Label: "Temperature",
+    Variable: "Temperature",
+    Unit: "°C",
+    Value: temperature,
+  }
+
+  (*attributes)["Status"] = values.Status
+  return nil
+}

src/udi/handlers/ttn/ttn.go

@@ -2,7 +2,7 @@ package ttn
 
 import (
   "fmt"
-  //"log"
+  "log"
   "time"
   "encoding/json"
   "udi/config"
@@ -14,6 +14,7 @@ import (
   "udi/handlers/ttn/models/draginoLsn50"
   "udi/handlers/ttn/models/rawPayloadPrinter"
   "udi/handlers/ttn/models/hottisScd30"
+  "udi/handlers/ttn/models/hottisGy21"
   "udi/handlers/ttn/models/hottisThreeWayThermometer"
   "udi/database"
 )
@@ -85,6 +86,7 @@ func New(id string, config config.HandlerConfigT) handler.Handler {
   }
   t.Id = id
   t.dbh = database.NewDatabaseHandle()
+  log.Printf("Handler TTN %d initialized", id)
   return t
 }
 
@@ -153,6 +155,8 @@ func (self *TTNHandler) Handle(message handler.MessageT) {
     parser = rawPayloadPrinter.Parse
   case "hottis-scd30":
     parser = hottisScd30.Parse
+  case "hottis-gy21":
+    parser = hottisGy21.Parse
   case "hottis-threeway-thermometer":
     parser = hottisThreeWayThermometer.Parse
   default:

src/udi/handlers/z2m/models/gs361ah04/gs361ah04.go

@@ -0,0 +1,15 @@
+package gs361ah04
+
+type Observation struct {
+  LinkQuality uint8 `unit:"" json:"linkquality"`
+  Battery uint8 `unit:"%" json:"battery"`
+  AwayMode string `unit:"" json:"away_mode"`
+  ChildLock string `unit:"" json:"child_lock"`
+  CurrentHeatingSetpoint float32 `unit:"°C" json:"current_heating_setpoint"`
+  LocalTemperature float32 `unit:"°C" json:"local_temperature"`
+  Preset string `unit:"" json:"preset"`
+  SystemMode string `unit:"" json:"system_mode"`
+  ValveDetection string `unit:"" json:"valve_detection"`
+  WindowDetection string `unit:"" json:"window_detection"`
+}
+

src/udi/handlers/z2m/models/wsdcgq01lm/wsdcgq01lm.go

@@ -0,0 +1,11 @@
+package wsdcgq01lm
+
+type Observation struct {
+  LinkQuality uint8 `unit:"" json:"linkquality"`
+  Battery uint8 `unit:"%" json:"battery"`
+  Humidity float32 `unit:"%H" json:"humidity"`
+  Pressure float32 `unit:"mbar" json:"pressure"`
+  Temperature float32 `unit:"°C" json:"temperature"`
+  Voltage uint16 `unit:"mV" json:"voltage"`
+}
+

src/udi/handlers/z2m/models/wsdcgq11lm/wsdcgq11lm.go

@@ -0,0 +1,10 @@
+package wsdcgq11lm
+
+type Observation struct {
+  LinkQuality uint8 `unit:"" json:"linkquality"`
+  Battery uint8 `unit:"%" json:"battery"`
+  Humidity float32 `unit:"%H" json:"humidity"`
+  Temperature float32 `unit:"°C" json:"temperature"`
+  Voltage uint16 `unit:"mV" json:"voltage"`
+}
+

src/udi/handlers/z2m/z2m.go

@@ -0,0 +1,113 @@
+package z2m
+
+import (
+  "fmt"
+  "log"
+  "time"
+  "strings"
+  "reflect"
+  "encoding/json"
+  "udi/config"
+  "udi/handlers/handler"
+  "udi/database"
+  "udi/handlers/z2m/models/wsdcgq11lm"
+  "udi/handlers/z2m/models/wsdcgq01lm"
+  "udi/handlers/z2m/models/gs361ah04"
+)
+
+
+type Z2MHandler struct {
+  handler.CommonHandler
+  dbh *database.DatabaseHandle
+}
+
+func parse(T any, payload string, variables *map[string]database.VariableType) error {
+  observationType := reflect.TypeOf(T)
+  observation := reflect.New(observationType).Interface()
+
+  err := json.Unmarshal([]byte(payload), observation)
+  if err != nil {
+    return fmt.Errorf("Unable to parse payload into Observation struct: %v, %s", err, payload)
+  }
+
+  observationValue := reflect.ValueOf(observation).Elem()
+
+  for i := 0; i < observationType.NumField(); i++ {
+    field := observationType.Field(i)
+    name := field.Name
+    unit := field.Tag.Get("unit")
+    value := observationValue.Field(i).Interface()
+
+    (*variables)[name] = database.VariableType {
+      Label: name,
+      Variable: "y",
+      Unit: unit,
+      Value: value,
+    }
+  }
+
+
+  return nil
+}
+
+func New(id string, config config.HandlerConfigT) handler.Handler {
+  t := &Z2MHandler {
+  }
+  t.Id = id
+  t.dbh = database.NewDatabaseHandle()
+  log.Printf("Handler Z2M %d initialized", id)
+  return t
+}
+
+func (self *Z2MHandler) Handle(message handler.MessageT) {
+  log.Printf("Handler Z2M %d processing %s -> %s", self.Id, message.Topic, message.Payload)
+
+  var measurement database.Measurement
+  measurement.Time = time.Now()
+
+
+  subTopics := strings.Split(message.Topic, "/")
+  deviceId := subTopics[1]
+  log.Printf("DeviceId: %s", deviceId)
+  device, err1 := self.dbh.GetDeviceByLabel(deviceId)
+  if err1 != nil {
+    self.Lost("Error when loading device", err1, message)
+    return
+  }
+  log.Printf("Device: %s", device)
+
+  measurement.Application = device.Application.Label
+  measurement.Device = device.Attributes["Label"].(string)
+
+  var T any
+  switch device.DeviceType.ModelIdentifier {
+  case "WSDCGQ11LM":
+    T = wsdcgq11lm.Observation{}
+  case "WSDCGQ01LM":
+    T = wsdcgq01lm.Observation{}
+  case "GS361A-H04":
+    T = gs361ah04.Observation{}
+  default:
+    self.Lost(fmt.Sprintf("No parser found for %s", device.DeviceType.ModelIdentifier), nil, message)
+    return
+  }
+
+  measurement.Values = make(map[string]database.VariableType)
+  measurement.Attributes = make(map[string]interface{})
+  err3 := parse(T, 
+                message.Payload,
+                &(measurement.Values))
+  if err3 != nil {
+    self.Lost("Model parser failed", err3, message)
+    return
+  }
+
+  measurement.Attributes["Status"] = "ok"
+  measurement.Attributes["DeviceId"] = deviceId
+  measurement.Attributes["DeviceModel"] = device.DeviceType.ModelIdentifier
+  log.Printf("Prepared measurement item: %s", measurement)
+  self.dbh.StoreMeasurement(&measurement)
+  self.S()
+}
+
+