Merge branch 'main' of gitea.hottis.de:wn/universal-data-ingest

.gitignore

@@ -1,4 +1,5 @@
 src/udi/udi
+src/udi/main
 src/udi/migrate_schema
 tmp/
 ENVDB

.woodpecker.yml

@@ -14,13 +14,6 @@ steps:
     when:
       - event: [push, tag]
 
-  scan_image:
-    image: aquasec/trivy
-    commands:
-      - trivy image $FORGE_NAME/$CI_REPO:$CI_COMMIT_SHA --quiet --exit-code 1
-    when:
-      - event: [push, tag]
-        
   deploy:
     image: portainer/kubectl-shell:latest
     secrets:

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.21-alpine as builder
+FROM golang:1.22.5-alpine3.20 as builder
 
 RUN mkdir -p /go/src
 COPY ./src/ /go/src

deployment/instances/udi/default/config.json

@@ -119,6 +119,16 @@
         }
       }
     },
+    {
+      "topics": [ "zigbee2mqtt/+" ],
+      "handler": "Z2M",
+      "id": "Z2M",
+      "config": {
+        "databaseConnStr": "",
+        "attributes": {
+        }
+      }
+    },
     {
       "topics": [ "shellyplusht/+/status/temperature:0" ],
       "handler": "SVEJ",

queries/hottis.sql

@@ -43,7 +43,22 @@ create or replace view temperature_v as
          cast(values->'Value'->>'value' as float) as temperature,
          device
     from measurements
-    where application in ('Temperature Multisensor', 'Temperature Shelly Plus HT');
+    where application in ('Temperature Multisensor', 'Temperature Shelly Plus HT')
+  union
+  select time,
+         cast(values->'Temperature'->>'value' as float) as temperature,
+         device
+    from measurements
+    where application = 'Zigbee2MQTT Hottis Eupenstr.' and
+          attributes->>'DeviceModel' in ('WSDCGQ11LM', 'WSDCGQ01LM');
+
+create or replace view voltage_v as
+  select time,
+         cast(values->'Voltage'->>'value' as float) / 1000 as voltage,
+         device
+    from measurements
+    where application = 'Zigbee2MQTT Hottis Eupenstr.' and
+          attributes->>'DeviceModel' in ('WSDCGQ11LM', 'WSDCGQ01LM');
 
 create or replace view temperature2_v as
   select time,
@@ -57,7 +72,14 @@ create or replace view humidity_v as
          cast(values->'Value'->>'value' as float) as humidity,
          device
     from measurements
-    where application in ('Humidity Multisensor');
+    where application in ('Humidity Multisensor')
+  union
+  select time,
+         cast(values->'Humidity'->>'value' as float) as temperature,
+         device
+    from measurements
+    where application = 'Zigbee2MQTT Hottis Eupenstr.' and
+          attributes->>'DeviceModel' in ('WSDCGQ11LM', 'WSDCGQ01LM');
 
 create or replace view soil_v as
   select time,
@@ -96,3 +118,14 @@ create or replace view router_v as
          cast(values->'wan-out'->>'value'  as int) as wanOutOctetsPerSeconds
     from measurements
     where application = 'SNMP' and device = '172.16.3.1';
+
+create or replace view lora_sht21_v as
+  select time,
+         cast(values->'Humidity'->>'value' as float) as humidity,
+         cast(values->'Temperature'->>'value' as float) as temperature,
+         m.device as device,
+         d.attributes->>'Label' as label
+    from measurements m, devices d
+    where m.application = 'de-hottis-app01' and 
+          m.attributes->>'DeviceType' = 'hottis-gy21' and
+          m.device = d.label;

queries/saerbeck.sql

@@ -9,3 +9,35 @@ create or replace view badesee_temperature_v as
     from measurements
     where application = 'de-hottis-saerbeck-monitoring' and
           device = 'eui-a84041318187ec13';
+
+create or replace view cubecell_threeway_temperature2_v as
+  select time,
+         cast(values->'Temperature2'->>'value' as float) as value,
+         values->'Temperature2'->>'label' as label
+    from measurements
+    where application = 'de-hottis-saerbeck-monitoring' and
+    device = 'eui-70b3d57ed0068fa4';
+
+create or replace view cubecell_threeway_temperature1_v as
+  select time,
+         cast(values->'Temperature1'->>'value' as float) as value,
+         values->'Temperature1'->>'label' as label
+    from measurements
+    where application = 'de-hottis-saerbeck-monitoring' and
+    device = 'eui-70b3d57ed0068fa4';
+
+create or replace view cubecell_threeway_temperature3_v as
+  select time,
+         cast(values->'Temperature3'->>'value' as float) as value,
+         values->'Temperature3'->>'label' as label
+    from measurements
+    where application = 'de-hottis-saerbeck-monitoring' and
+    device = 'eui-70b3d57ed0068fa4';
+
+create or replace view cubecell_threeway_battery_v as
+  select time,
+         cast(values->'Battery'->>'value' as float) as value,
+         values->'Battery'->>'label' as label
+    from measurements
+    where application = 'de-hottis-saerbeck-monitoring' and
+    device = 'eui-70b3d57ed0068fa4';

src/udi/ENVDB.uditest

@@ -1,6 +1,6 @@
-PGUSER="uditest"
+PGUSER="udi-hottis"
 PGHOST=`kubectl get services traefik -n system -o jsonpath="{.status.loadBalancer.ingress[0].ip}"`
-PGPASSWORD=`kubectl get secrets uditest-db-cred -n udi-test -o jsonpath="{.data.PGPASSWORD}" | base64 --decode`
+PGPASSWORD=`kubectl get secrets default-udi-db-cred -n udi -o jsonpath="{.data.PGPASSWORD}" | base64 --decode`
 PGSSLMODE=require
 PGDATABASE="uditest"
 export PGUSER PGHOST PGPASSWORD PGSSLMODE PGDATABASE

src/udi/config-test-ttn.json

@@ -1,7 +1,7 @@
 {
   "mqtt": {
     "broker": "ssl://eu1.cloud.thethings.network:8883",
-    "username": "de-hottis-lora-test1@ttn",
+    "username": "de-hottis-saerbeck-monitoring@ttn",
     "tlsEnable": "true"
   },
   "topicMappings": [

src/udi/config-test.json

@@ -4,72 +4,6 @@
     "tlsEnable": "false"
   },
   "topicMappings": [
-    {
-      "topics": [ "IoT/PV/Values" ],
-      "handler": "PV",
-      "id": "PV",
-      "config": {
-        "attributes": {
-        }
-      }
-    },
-    {
-      "topics": [ "IoT/MBGW3/Measurement" ],
-      "handler": "MBGW3",
-      "id": "MBGW3",
-      "config": {
-        "attributes": {
-        }
-      }
-    },
-    {
-      "topics": [ "dt1/ai/periodic/1" ],
-      "handler": "DT1T",
-      "id": "DT1T.0",
-      "config": {
-        "attributes": {
-          "Application": "Temperature Wago",
-          "Device": "Freezer",
-          "HardLow": "-273",
-          "SoftLow": "-50",
-          "SoftHigh": "20",
-          "HardHigh": "100"
-        }
-      }
-    },
-    {
-      "topics": [ "dt1/ai/periodic/3" ],
-      "handler": "DT1T",
-      "id": "DT1T.1",
-      "config": {
-        "attributes": {
-          "Application": "Temperature Wago",
-          "Device": "Outdoor",
-          "HardLow": "-273",
-          "SoftLow": "-60",
-          "SoftHigh": "60",
-          "HardHigh": "100"
-        }
-      }
-    },
-    {
-      "topics": [ "IoT/OneWireGW/Bus 1/#" ],
-      "handler": "SVER",
-      "id": "SVER0",
-      "config": {
-        "databaseConnStr": "",
-        "attributes": {
-          "application": "Temperature Heating",
-          "payloadRegex": "(\\d+(\\.\\d+)?)\\s*([^0-9\\s]\\S*)",
-          "deviceFrom": "topic", 
-          "devicePart": "3",
-          "valueFrom": "payload",
-          "valuePart": "1",
-          "unitFrom": "payload",
-          "unitPart": "3"
-        }
-      }
-    },
     {
       "topics": [ "NR/Multisensor/+/Temperatur" ],
       "handler": "SVEJ",
@@ -85,44 +19,12 @@
       }
     },
     {
-      "topics": [ "NR/Multisensor/+/Feuchte" ],
+      "topics": [ "zigbee2mqtt/+" ],
-      "handler": "SVEJ",
+      "handler": "Z2M",
-      "id": "SVEJ1",
+      "id": "Z2M",
       "config": {
         "databaseConnStr": "",
         "attributes": {
-          "application": "Humidity Multisensor",
-          "deviceSelector": "T:2",
-          "valueSelector": "J:$.CurrentRelativeHumidity",
-          "unitSelector": "C:%"
-        }
-      }
-    },
-    {
-      "topics": [ "shellyplusht/+/status/temperature:0" ],
-      "handler": "SVEJ",
-      "id": "SVEJ2",
-      "config": {
-        "databaseConnStr": "",
-        "attributes": {
-          "application": "Temperature Shelly Plus HT",
-          "deviceSelector": "T:1",
-          "valueSelector": "J:$.tC",
-          "unitSelector": "C:°C"
-        }
-      }
-    },
-    {
-      "topics": [ "shellyplusht/+/status/humidity:0" ],
-      "handler": "SVEJ",
-      "id": "SVE4",
-      "config": {
-        "databaseConnStr": "",
-        "attributes": {
-          "application": "Humidity Shelly Plus HT",
-          "deviceSelector": "T:1",
-          "valueSelector": "J:$.rh",
-          "unitSelector": "C:%"
         }
       }
     }

src/udi/database/database.go

@@ -25,7 +25,7 @@ func NewDatabaseHandle() *DatabaseHandle {
   } else {
     db.dbh = conn
     db.initialized = true
-    //log.Println("Database connection opened")
+    log.Println("Database connection opened")
   }
   return &db
 }
@@ -44,7 +44,7 @@ func (self *DatabaseHandle) StoreMeasurement(measurement *Measurement) {
     return
   }
 
-  //log.Println("Successfully stored measurement")
+  log.Println("Successfully stored measurement")
   counter.S("Stored")
 }
 
@@ -70,5 +70,27 @@ func (self *DatabaseHandle) GetDeviceByLabelAndApplication(applicationLabel stri
   return &device, nil
 }
 
+func (self *DatabaseHandle) GetDeviceByLabel(deviceLabel string) (*Device, error) {
+  if ! self.initialized {
+    err := fmt.Errorf("Database connection not initialized")
+    return nil, err
+  }
+
+  var device Device
+  result := self.dbh.
+    Preload("Application").
+    Preload("DeviceType").
+    Where("devices.label = ?", deviceLabel).
+    First(&device)
+
+  if result.Error != nil {
+    err := fmt.Errorf("Query failed: %s", result.Error)
+    return nil, err
+  }
+
+  return &device, nil
+}
+
+
 
 

src/udi/dispatcher/dispatcher.go

@@ -18,6 +18,7 @@ import "udi/handlers/svej"
 import "udi/handlers/dt1t"
 import "udi/handlers/locative"
 import "udi/handlers/snmp"
+import "udi/handlers/z2m"
 
 
 var handlerMap map[string]handler.Handler = make(map[string]handler.Handler)
@@ -50,6 +51,8 @@ func InitDispatcher() {
       factory = locative.New
     case "SNMP":
       factory = snmp.New
+    case "Z2M":
+      factory = z2m.New
     default:
       factory = nil
       log.Printf("No handler %s found, ignore mapping", mapping.Handler)

src/udi/go.mod

@@ -1,24 +1,25 @@
 module udi
 
-go 1.21.3
+go 1.22.3
 
 require (
 	github.com/eclipse/paho.mqtt.golang v1.4.3
 	github.com/google/uuid v1.4.0
 	github.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852
-	gorm.io/driver/postgres v1.5.4
+	gorm.io/driver/postgres v1.5.9
-	gorm.io/gorm v1.25.5
+	gorm.io/gorm v1.25.11
 )
 
 require (
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
-	github.com/jackc/pgx/v5 v5.4.3 // indirect
+	github.com/jackc/pgx/v5 v5.5.5 // indirect
+	github.com/jackc/puddle/v2 v2.2.1 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
-	golang.org/x/crypto v0.19.0 // indirect
+	golang.org/x/crypto v0.25.0 // indirect
-	golang.org/x/net v0.20.0 // indirect
+	golang.org/x/net v0.27.0 // indirect
-	golang.org/x/sync v0.1.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
 )

src/udi/go.sum

@@ -11,8 +11,10 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
 github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
 github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
-github.com/jackc/pgx/v5 v5.4.3 h1:cxFyXhxlvAifxnkKKdlxv8XqUf59tDlYjnV5YYfsJJY=
+github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw=
-github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA=
+github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A=
+github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
+github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
@@ -26,25 +28,19 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
+golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
-golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
-golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
+golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
-golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
-golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
-golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
-golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
-golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
-golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gorm.io/driver/postgres v1.5.4 h1:Iyrp9Meh3GmbSuyIAGyjkN+n9K+GHX9b9MqsTL4EJCo=
+gorm.io/driver/postgres v1.5.9 h1:DkegyItji119OlcaLjqN11kHoUgZ/j13E0jkJZgD6A8=
-gorm.io/driver/postgres v1.5.4/go.mod h1:Bgo89+h0CRcdA33Y6frlaHHVuTdOf87pmyzwW9C/BH0=
+gorm.io/driver/postgres v1.5.9/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI=
-gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls=
+gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg=
-gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
+gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ=

src/udi/handlers/dt1t/dt1t.go

@@ -38,6 +38,7 @@ func New(id string, config config.HandlerConfigT) handler.Handler {
   t.Id = id
   
   t.dbh = database.NewDatabaseHandle()
+  log.Printf("Handler DT1T %d initialized", id)
   t.ready = true
   return t
 }

src/udi/handlers/locative/locative.go

@@ -31,6 +31,7 @@ func New(id string, config config.HandlerConfigT) handler.Handler {
   }
   t.Id = id
   t.dbh = database.NewDatabaseHandle()
+  log.Printf("Handler Locative %d initialized", id)
   return t
 }
 

src/udi/handlers/mbgw3/mbgw3.go

@@ -2,6 +2,7 @@ package mbgw3
 
 import (
   "time"
+  "log"
   "strconv"
   "encoding/json"
   "udi/config"
@@ -31,6 +32,7 @@ func New(id string, config config.HandlerConfigT) handler.Handler {
   }
   t.Id = id
   t.dbh = database.NewDatabaseHandle()
+  log.Printf("Handler MBGW3 %d initialized", id)
   return t
 }
 

src/udi/handlers/pv/pv.go

@@ -3,6 +3,7 @@ package pv
 import (
   "reflect"
   "time"
+  "log"
   "encoding/json"
   "udi/config"
   "udi/handlers/handler"
@@ -41,6 +42,7 @@ func New(id string, config config.HandlerConfigT) handler.Handler {
   }
   t.Id = id
   t.dbh = database.NewDatabaseHandle()
+  log.Printf("Handler PV %d initialized", id)
   return t
 }
 

src/udi/handlers/snmp/snmp.go

@@ -33,11 +33,12 @@ func New(id string, config config.HandlerConfigT) handler.Handler {
   }
   t.Id = id
   t.dbh = database.NewDatabaseHandle()
+  log.Printf("Handler SNMP %d initialized", id)
   return t
 }
 
 func (self *SnmpHandler) Handle(message handler.MessageT) {
-  log.Printf("Handler SNMP %d processing %s -> %s", self.Id, message.Topic, message.Payload)
+  // log.Printf("Handler SNMP %d processing %s -> %s", self.Id, message.Topic, message.Payload)
 
   var observation observation_t
   err := json.Unmarshal([]byte(message.Payload), &observation)
@@ -66,7 +67,7 @@ func (self *SnmpHandler) Handle(message handler.MessageT) {
     }
   }
 
-  log.Printf("Prepared measurement item: %s", measurement)
+  // log.Printf("Prepared measurement item: %s", measurement)
 
   self.dbh.StoreMeasurement(&measurement)
   self.S()

src/udi/handlers/svej/svej.go

@@ -76,19 +76,22 @@ func New(id string, config config.HandlerConfigT) handler.Handler {
   t.Id = id
   t.ready = true
   t.dbh = database.NewDatabaseHandle()
+  log.Printf("Handler SVEJ %d initialized", id)
   return t
 }
 
-func extractionHelper(subTopics []string, jPayload interface{}, selector string, jp *jsonpath.Compiled) (string, error) {
+func (self *SingleValueExtractorJsonpathHandler) ExtractionHelper(subTopics []string, jPayload interface{}, selector string, jp *jsonpath.Compiled) (string, error) {
   var res string
   switch selector[:2] {
   case "J:":
+    // extract using jsonpath from payload
     r, e := jp.Lookup(jPayload)
     if e != nil {
       return "", fmt.Errorf("jp.Lookup failed with %s", e)
     }
     res = fmt.Sprint(r)
   case "T:":
+    // T: extract from topic
     i, e := strconv.Atoi(selector[2:])
     if e != nil {
       return "", fmt.Errorf("Atoi failed with %s", e)
@@ -98,6 +101,7 @@ func extractionHelper(subTopics []string, jPayload interface{}, selector string,
     }
     res = subTopics[i]
   case "C:":
+    // use constant value
     res = selector[2:]
   default:
     return "", fmt.Errorf("Invalid selector: %s", selector[:2])
@@ -111,14 +115,14 @@ func (self *SingleValueExtractorJsonpathHandler) Handle(message handler.MessageT
     self.Lost("Handler is not marked as ready", nil, message)
     return
   }
-  //log.Printf("Handler SingleValueExtractorJsonpath %d processing %s -> %s", self.Id, message.Topic, message.Payload)
+  log.Printf("Handler SingleValueExtractorJsonpath %d processing %s -> %s", self.Id, message.Topic, message.Payload)
 
   var measurement database.Measurement
   measurement.Time = time.Now()
   measurement.Application = self.application
   
   subTopics := strings.Split(message.Topic, "/")
-  //log.Printf("Subtopics: %s", strings.Join(subTopics, ", "))
+  log.Printf("Subtopics: %s", strings.Join(subTopics, ", "))
   var jPayload interface{}
   err := json.Unmarshal([]byte(message.Payload), &jPayload)
   if err != nil {
@@ -126,17 +130,20 @@ func (self *SingleValueExtractorJsonpathHandler) Handle(message handler.MessageT
     return
   }
 
-  device, err1 := extractionHelper(subTopics, jPayload, self.deviceSelector, self.deviceJsonpath)
+  device, err1 := self.ExtractionHelper(subTopics, jPayload, self.deviceSelector, self.deviceJsonpath)
   if err1 != nil {
     self.Lost("Device extraction failed", err1, message)
     return
   }
-  value, err2 := extractionHelper(subTopics, jPayload, self.valueSelector, self.valueJsonpath)
+  log.Printf("device: %s", device) 
+
+  value, err2 := self.ExtractionHelper(subTopics, jPayload, self.valueSelector, self.valueJsonpath)
   if err2 != nil {
     self.Lost("Value extraction failed", err2, message)
     return
   }
-  unit, err3 := extractionHelper(subTopics, jPayload, self.unitSelector, self.unitJsonpath)
+
+  unit, err3 := self.ExtractionHelper(subTopics, jPayload, self.unitSelector, self.unitJsonpath)
   if err3 != nil {
     self.Lost("Unit extraction failed", err3, message)
     return
@@ -152,7 +159,7 @@ func (self *SingleValueExtractorJsonpathHandler) Handle(message handler.MessageT
   measurement.Values = make(map[string]database.VariableType)
   measurement.Values["Value"] = variable
 
-  //log.Printf("Prepared measurement item: %s", measurement)
+  log.Printf("Prepared measurement item: %s", measurement)
   self.dbh.StoreMeasurement(&measurement)
   self.S()
 }

src/udi/handlers/sver/sver.go

@@ -111,6 +111,7 @@ func New(id string, config config.HandlerConfigT) handler.Handler {
   t.Id = id
   t.ready = true
   t.dbh = database.NewDatabaseHandle()
+  log.Printf("Handler SVER %d initialized", id)
   return t
 }
 

src/udi/handlers/ttn/models/hottisGy21/hottisGY21.go

@@ -0,0 +1,55 @@
+package hottisGy21
+
+import (
+  //"log"
+  "fmt"
+  "bytes"
+  "encoding/base64"
+  "encoding/binary"
+  "udi/database"
+)
+
+type hottisGy21Values struct {
+  Connected uint8
+  Status uint8
+  RawHumidity uint16
+  RawTemperature uint16
+}
+
+
+func Parse(fPort int, _ []byte, frmPayload string, variables *map[string]database.VariableType, attributes *map[string]interface{}, _ *database.Device) error {
+  if fPort != 2 {
+    return fmt.Errorf("Unexpected fPort %d", fPort)
+  }
+
+  b, err := base64.StdEncoding.DecodeString(frmPayload)
+  if err != nil {
+    return fmt.Errorf("Unable to base64-decode payload: %v", err)
+  }
+
+  var values hottisGy21Values
+  err = binary.Read(bytes.NewReader(b), binary.LittleEndian, &values)
+  if err != nil {
+    return fmt.Errorf("Unable to cast into struct: %v", err)
+  }
+  var temperature float32 = -46.85 + 175.72 * (float32(values.RawTemperature) / 65536.0)
+  var humidity float32 = -6 + 125 * (float32(values.RawHumidity) / 65536.0);
+
+  // log.Printf("CO2: %f, Temp: %f, Hum: %f, Status: %d", co2concentration, temperature, humidity, values.Status)
+
+  (*variables)["Humidity"] = database.VariableType {
+    Label: "Humidity",
+    Variable: "Humidity",
+    Unit: "%",
+    Value: humidity,
+  }
+  (*variables)["Temperature"] = database.VariableType {
+    Label: "Temperature",
+    Variable: "Temperature",
+    Unit: "°C",
+    Value: temperature,
+  }
+
+  (*attributes)["Status"] = values.Status
+  return nil
+}

src/udi/handlers/ttn/models/hottisThreeWayThermometer/hottisThreeWayThermometer.go

@@ -0,0 +1,93 @@
+package hottisThreeWayThermometer
+
+import (
+  "log"
+  "fmt"
+  "bytes"
+  "strconv"
+  "encoding/base64"
+  "encoding/binary"
+  "udi/database"
+)
+
+type hottisThreeWayThermometerValues struct {
+  Status uint8
+  Battery uint16
+  SensorAddress1 uint64
+  Value1 int32
+  SensorAddress2 uint64
+  Value2 int32
+  SensorAddress3 uint64
+  Value3 int32 
+}
+
+func getSensorName(sensorsMap *map[string]interface{}, sensorAddress uint64) string {
+  key := strconv.FormatUint(sensorAddress, 10)
+	if sensorName, exists := (*sensorsMap)[key].(string); exists {
+		return sensorName
+	}
+	return "Sensor" + key
+}
+
+func Parse(fPort int, _ []byte, frmPayload string, variables *map[string]database.VariableType, attributes *map[string]interface{}, device *database.Device) error {
+  deviceAttrs := (*device).Attributes
+  sensorsMap := deviceAttrs["Sensors"].(map[string]interface{})
+
+  if fPort != 2 {
+    return fmt.Errorf("Unexpected fPort %d", fPort)
+  }
+
+  b, err := base64.StdEncoding.DecodeString(frmPayload)
+  if err != nil {
+    return fmt.Errorf("Unable to base64-decode payload: %v", err)
+  }
+
+  var values hottisThreeWayThermometerValues
+  err = binary.Read(bytes.NewReader(b), binary.LittleEndian, &values)
+  if err != nil {
+    return fmt.Errorf("Unable to cast into struct: %v", err)
+  }
+
+  var battery float32 = float32(values.Battery) / 1000.0
+  var value1 float32 = float32(values.Value1) / 128.0
+  var value2 float32 = float32(values.Value2) / 128.0
+  var value3 float32 = float32(values.Value3) / 128.0
+
+  //log.Printf("Status: %d, Battery: %d", values.Status, values.Battery);
+  log.Printf("Status: %d", values.Status);
+  log.Printf("Battery: %d, %f", values.Battery, battery);
+  log.Printf("Sensor1: Addr: %d, Value: %f", values.SensorAddress1, value1);
+  log.Printf("Sensor2: Addr: %d, Value: %f", values.SensorAddress2, value2);
+  log.Printf("Sensor3: Addr: %d, Value: %f", values.SensorAddress3, value3);
+
+  (*variables)["Battery"] = database.VariableType {
+    Label: "Battery",
+    Variable: "Voltage",
+    Unit: "V",
+    Value: battery,
+  }
+  (*variables)["Temperature1"] = database.VariableType {
+    Label: getSensorName(&sensorsMap, values.SensorAddress1),
+    Variable: "Temperature",
+    Unit: "°C",
+    Value: value1,
+  }
+  (*variables)["Temperature2"] = database.VariableType {
+    Label: getSensorName(&sensorsMap, values.SensorAddress2),
+    Variable: "Temperature",
+    Unit: "°C",
+    Value: value2,
+  }
+  (*variables)["Temperature3"] = database.VariableType {
+    Label: getSensorName(&sensorsMap, values.SensorAddress3),
+    Variable: "Temperature",
+    Unit: "°C",
+    Value: value3,
+  }
+
+  (*attributes)["Status"] = values.Status
+  (*attributes)["SensorAddress1"] = values.SensorAddress1
+  (*attributes)["SensorAddress2"] = values.SensorAddress2
+  (*attributes)["SensorAddress3"] = values.SensorAddress3
+  return nil
+}

src/udi/handlers/ttn/ttn.go

@@ -2,7 +2,7 @@ package ttn
 
 import (
   "fmt"
-  //"log"
+  "log"
   "time"
   "encoding/json"
   "udi/config"
@@ -14,6 +14,8 @@ import (
   "udi/handlers/ttn/models/draginoLsn50"
   "udi/handlers/ttn/models/rawPayloadPrinter"
   "udi/handlers/ttn/models/hottisScd30"
+  "udi/handlers/ttn/models/hottisGy21"
+  "udi/handlers/ttn/models/hottisThreeWayThermometer"
   "udi/database"
 )
 
@@ -84,6 +86,7 @@ func New(id string, config config.HandlerConfigT) handler.Handler {
   }
   t.Id = id
   t.dbh = database.NewDatabaseHandle()
+  log.Printf("Handler TTN %d initialized", id)
   return t
 }
 
@@ -152,6 +155,10 @@ func (self *TTNHandler) Handle(message handler.MessageT) {
     parser = rawPayloadPrinter.Parse
   case "hottis-scd30":
     parser = hottisScd30.Parse
+  case "hottis-gy21":
+    parser = hottisGy21.Parse
+  case "hottis-threeway-thermometer":
+    parser = hottisThreeWayThermometer.Parse
   default:
     self.Lost(fmt.Sprintf("No parser found for %s", device.DeviceType.ModelIdentifier), nil, message)
     return

src/udi/handlers/z2m/models/gs361ah04/gs361ah04.go

@@ -0,0 +1,15 @@
+package gs361ah04
+
+type Observation struct {
+  LinkQuality uint8 `unit:"" json:"linkquality"`
+  Battery uint8 `unit:"%" json:"battery"`
+  AwayMode string `unit:"" json:"away_mode"`
+  ChildLock string `unit:"" json:"child_lock"`
+  CurrentHeatingSetpoint float32 `unit:"°C" json:"current_heating_setpoint"`
+  LocalTemperature float32 `unit:"°C" json:"local_temperature"`
+  Preset string `unit:"" json:"preset"`
+  SystemMode string `unit:"" json:"system_mode"`
+  ValveDetection string `unit:"" json:"valve_detection"`
+  WindowDetection string `unit:"" json:"window_detection"`
+}
+

src/udi/handlers/z2m/models/wsdcgq01lm/wsdcgq01lm.go

@@ -0,0 +1,11 @@
+package wsdcgq01lm
+
+type Observation struct {
+  LinkQuality uint8 `unit:"" json:"linkquality"`
+  Battery uint8 `unit:"%" json:"battery"`
+  Humidity float32 `unit:"%H" json:"humidity"`
+  Pressure float32 `unit:"mbar" json:"pressure"`
+  Temperature float32 `unit:"°C" json:"temperature"`
+  Voltage uint16 `unit:"mV" json:"voltage"`
+}
+

src/udi/handlers/z2m/models/wsdcgq11lm/wsdcgq11lm.go

@@ -0,0 +1,10 @@
+package wsdcgq11lm
+
+type Observation struct {
+  LinkQuality uint8 `unit:"" json:"linkquality"`
+  Battery uint8 `unit:"%" json:"battery"`
+  Humidity float32 `unit:"%H" json:"humidity"`
+  Temperature float32 `unit:"°C" json:"temperature"`
+  Voltage uint16 `unit:"mV" json:"voltage"`
+}
+

src/udi/handlers/z2m/z2m.go

@@ -0,0 +1,113 @@
+package z2m
+
+import (
+  "fmt"
+  "log"
+  "time"
+  "strings"
+  "reflect"
+  "encoding/json"
+  "udi/config"
+  "udi/handlers/handler"
+  "udi/database"
+  "udi/handlers/z2m/models/wsdcgq11lm"
+  "udi/handlers/z2m/models/wsdcgq01lm"
+  "udi/handlers/z2m/models/gs361ah04"
+)
+
+
+type Z2MHandler struct {
+  handler.CommonHandler
+  dbh *database.DatabaseHandle
+}
+
+func parse(T any, payload string, variables *map[string]database.VariableType) error {
+  observationType := reflect.TypeOf(T)
+  observation := reflect.New(observationType).Interface()
+
+  err := json.Unmarshal([]byte(payload), observation)
+  if err != nil {
+    return fmt.Errorf("Unable to parse payload into Observation struct: %v, %s", err, payload)
+  }
+
+  observationValue := reflect.ValueOf(observation).Elem()
+
+  for i := 0; i < observationType.NumField(); i++ {
+    field := observationType.Field(i)
+    name := field.Name
+    unit := field.Tag.Get("unit")
+    value := observationValue.Field(i).Interface()
+
+    (*variables)[name] = database.VariableType {
+      Label: name,
+      Variable: "y",
+      Unit: unit,
+      Value: value,
+    }
+  }
+
+
+  return nil
+}
+
+func New(id string, config config.HandlerConfigT) handler.Handler {
+  t := &Z2MHandler {
+  }
+  t.Id = id
+  t.dbh = database.NewDatabaseHandle()
+  log.Printf("Handler Z2M %d initialized", id)
+  return t
+}
+
+func (self *Z2MHandler) Handle(message handler.MessageT) {
+  log.Printf("Handler Z2M %d processing %s -> %s", self.Id, message.Topic, message.Payload)
+
+  var measurement database.Measurement
+  measurement.Time = time.Now()
+
+
+  subTopics := strings.Split(message.Topic, "/")
+  deviceId := subTopics[1]
+  log.Printf("DeviceId: %s", deviceId)
+  device, err1 := self.dbh.GetDeviceByLabel(deviceId)
+  if err1 != nil {
+    self.Lost("Error when loading device", err1, message)
+    return
+  }
+  log.Printf("Device: %s", device)
+
+  measurement.Application = device.Application.Label
+  measurement.Device = device.Attributes["Label"].(string)
+
+  var T any
+  switch device.DeviceType.ModelIdentifier {
+  case "WSDCGQ11LM":
+    T = wsdcgq11lm.Observation{}
+  case "WSDCGQ01LM":
+    T = wsdcgq01lm.Observation{}
+  case "GS361A-H04":
+    T = gs361ah04.Observation{}
+  default:
+    self.Lost(fmt.Sprintf("No parser found for %s", device.DeviceType.ModelIdentifier), nil, message)
+    return
+  }
+
+  measurement.Values = make(map[string]database.VariableType)
+  measurement.Attributes = make(map[string]interface{})
+  err3 := parse(T, 
+                message.Payload,
+                &(measurement.Values))
+  if err3 != nil {
+    self.Lost("Model parser failed", err3, message)
+    return
+  }
+
+  measurement.Attributes["Status"] = "ok"
+  measurement.Attributes["DeviceId"] = deviceId
+  measurement.Attributes["DeviceModel"] = device.DeviceType.ModelIdentifier
+  log.Printf("Prepared measurement item: %s", measurement)
+  self.dbh.StoreMeasurement(&measurement)
+  self.S()
+}
+
+