queries

.woodpecker.yml

@@ -1,5 +1,52 @@
 steps:
   build:
+    image: golang:1.22.5-alpine3.20
+    commands:
+      - GOPATH=/woodpecker/go
+      - cd src/udi
+      - go mod tidy
+      - go build -a -installsuffix nocgo -o udi main.go
+      - cp udi ../..
+    when:
+      - event: [push, tag]
+
+  scan:
+    image: quay.io/wollud1969/woodpecker-helper:0.5.1
+    environment:
+      TRIVY_TOKEN:
+        from_secret: trivy_token
+      TRIVY_URL:
+        from_secret: trivy_url
+      DTRACK_API_KEY:
+        from_secret: dtrack_api_key
+      DTRACK_API_URL:
+        from_secret: dtrack_api_url
+    commands:
+      - export GOPATH=/woodpecker/go   # the export is required, otherwise trivy will not consider the variable
+      - HOME=/home/`id -nu`
+      - TAG="${CI_COMMIT_TAG:-$CI_COMMIT_SHA}"
+      - |
+        trivy fs \
+              --server $TRIVY_URL \
+              --token $TRIVY_TOKEN \
+              --format cyclonedx \
+              --scanners license \
+              --output /tmp/sbom.xml \
+              .        
+      - cat /tmp/sbom.xml
+      - |
+        curl -X "POST" \
+             -H "Content-Type: multipart/form-data" \
+             -H "X-Api-Key: $DTRACK_API_KEY" \
+             -F "autoCreate=true" \
+             -F "projectName=$CI_REPO" \
+             -F "projectVersion=$TAG" \
+             -F "bom=@/tmp/sbom.xml"\
+             "$DTRACK_API_URL/api/v1/bom"        
+    when:
+      - event: [push, tag]
+
+  dockerize:
     image: plugins/kaniko
     settings:
       repo: ${FORGE_NAME}/${CI_REPO}
@@ -15,14 +62,14 @@ steps:
       - event: [push, tag]
 
   deploy:
-    image: portainer/kubectl-shell:latest
+    image: quay.io/wollud1969/woodpecker-helper:0.5.1
-    secrets:
+    environment:
-      - source: kube_config
+      KUBE_CONFIG_CONTENT:
-        target: KUBE_CONFIG_CONTENT
+        from_secret: kube_config
-      - source: encryption_key
+      ENCRYPTION_KEY:
-        target: ENCRYPTION_KEY
+        from_secret: encryption_key
-      - source: secrets_checksum
+      MD5_CHECKSUM:
-        target: MD5_CHECKSUM
+        from_secret: secrets_checksum
     commands:
       - export IMAGE_TAG=$CI_COMMIT_TAG
       - printf "$KUBE_CONFIG_CONTENT" > /tmp/kubeconfig

Dockerfile

@@ -1,15 +1,8 @@
-FROM golang:1.22.5-alpine3.20 as builder
-
-RUN mkdir -p /go/src
-COPY ./src/ /go/src
-WORKDIR /go/src/udi
-RUN go build -a -installsuffix nocgo -o udi main.go
-
-
 FROM scratch
 
 ENV UDI_CONF ""
 
-COPY --from=builder /go/src/udi ./
+COPY udi ./
+
 ENTRYPOINT ["./udi"]
 

deployment/instances/udi/default/config.json

@@ -6,13 +6,22 @@
   "topicMappings": [
     {
       "topics": [ "snmp" ],
-      "handler": "SNMP",
+      "handler": "PREP",
       "id": "SNMP",
       "config": {
         "attributes": {
         }
       }
     },
+    {
+      "topics": [ "tsm" ],
+      "handler": "PREP",
+      "id": "TSM",
+      "config": {
+        "attributes": {
+        }
+      }
+    },
     {
       "topics": [ "dt1/ai/periodic/1" ],
       "handler": "DT1T",

migration/.gitignore

@@ -1 +0,0 @@
-.venv

migration/migrate-level.py

@@ -1,38 +0,0 @@
-import psycopg2
-from loguru import logger
-
-try:
-    srcConn = psycopg2.connect(database="level_monitoring_berresheim")
-    srcConn.autocommit = False
-    destConn = psycopg2.connect(database="udi-berresheim")
-    destConn.autocommit = False
-
-    with srcConn.cursor() as srcCur, destConn.cursor() as destCur:
-        srcCur.execute("select time, application_name, raw_level, level, status, battery from measurement_t")
-        for srcObj in srcCur:
-            timestamp = srcObj[0]
-            deviceName = srcObj[1]
-            rawLevel = srcObj[2]
-            level = srcObj[3]
-            status = srcObj[4]
-            battery = srcObj[5]
-
-            logger.info(f"{timestamp=}, {deviceName=}, {rawLevel=}, {level=}, {status=}, {battery=}")
-
-            destTime = timestamp
-            destApplication = "de-hottis-level-monitoring"
-            destDevice = "eui-a84041a2c18341d6"
-            destAttributes = '{"ApplicationId":"de-hottis-level-monitoring", "DeviceType":"dragino-ldds75", "Status":"' + status + '","Hint": "Migrated"}'
-            destValues = '{"Battery":{"unit":"V","label":"Battery","value":' + str(battery) + ',"variable":"Voltage"}, "Distance":{"unit":"mm","label":"Distance","variable":"Level","value":' + str(rawLevel) + '}, "CorrectedDistance":{"unit":"mm", "label":"CorrectedDistance", "variable":"Level","value":' + str(level) + '}}'
-            logger.info(f"{destTime=}, {destApplication=}, {destDevice=}, {destAttributes=}, {destValues=}")
-
-            destCur.execute("insert into measurements (time, application, device, attributes, values) values(%s, %s, %s, %s, %s)",
-                            (destTime, destApplication, destDevice, destAttributes, destValues))
-            destConn.commit()
-finally:
-    if srcConn:
-        srcConn.close()
-    if destConn:
-        destConn.close()
-
-

migration/migrate-pv.py

@@ -1,79 +0,0 @@
-import psycopg2
-from loguru import logger
-import os
-
-srcPgHost = os.environ["SRC_PGHOST"]
-srcPgUser = os.environ["SRC_PGUSER"]
-srcPgPassword = os.environ["SRC_PGPASSWORD"]
-srcPgDatabase = os.environ["SRC_PGDATABASE"]
-destPgHost = os.environ["DEST_PGHOST"]
-destPgUser = os.environ["DEST_PGUSER"]
-destPgPassword = os.environ["DEST_PGPASSWORD"]
-destPgDatabase = os.environ["DEST_PGDATABASE"]
-
-try:
-    srcConn = psycopg2.connect(
-                host=srcPgHost,
-                dbname=srcPgDatabase,
-                user=srcPgUser,
-                password=srcPgPassword,
-                sslmode='require'
-            )
-    srcConn.autocommit = False
-
-    destConn = psycopg2.connect(
-                host=destPgHost,
-                dbname=destPgDatabase,
-                user=destPgUser,
-                password=destPgPassword,
-                sslmode='require'
-            )
-    destConn.autocommit = False
-
-    with srcConn.cursor() as srcCur, destConn.cursor() as destCur:
-        srcCur.execute("select time, deviceid, status, state, importenergyactive, importenergyreactive, exportenergyactive, exportenergyreactive, powerapparent, poweractive, powerreactive, powerdemandpositive, powerdemandreverse, factor, angle, voltage, current, powerdemand from pv_power_measurement_t order by time")
-        for srcObj in srcCur:
-            timestamp = srcObj[0]
-            deviceName = srcObj[1]
-            status = srcObj[2]
-            state = srcObj[3]
-            importenergyactive = srcObj[4]
-            importenergyreactive = srcObj[5]
-            exportenergyactive = srcObj[6]
-            exportenergyreactive = srcObj[7]
-            powerapparent = srcObj[8]
-            poweractive = srcObj[9]
-            powerreactive = srcObj[10]
-            powerdemandpositive = srcObj[11]
-            powerdemandreverse = srcObj[12]
-            factor = srcObj[13]
-            angle = srcObj[14]
-            voltage = srcObj[15]
-            current = srcObj[16]
-            powerdemand = srcObj[17]
-
-
-            logger.info(f"{timestamp=}, {deviceName=}")
-
-            destTime = timestamp
-            destApplication = "PV"
-            destDevice = "Powermeter"
-            destAttributes = f"{{\"ApplicationId\":\"PV\", \"Status\":\"{status}\",\"Hint\": \"Migrated\"}}"
-            destValues = f"{{\"Cnt\": {{\"unit\": \"\", \"label\": \"\", \"value\": \"-1\", \"variable\": \"Cnt\"}}, \"Angle\": {{\"unit\": \"degree\", \"label\": \"\", \"value\": \"{angle}\", \"variable\": \"Angle\"}}, \"State\": {{\"unit\": \"\", \"label\": \"\", \"value\": \"{state}\", \"variable\": \"State\"}}, \"Factor\": {{\"unit\": \"\", \"label\": \"\", \"value\": \"{factor}\", \"variable\": \"Factor\"}}, \"Current\": {{\"unit\": \"A\", \"label\": \"\", \"value\": \"{current}\", \"variable\": \"Current\"}}, \"Voltage\": {{\"unit\": \"V\", \"label\": \"\", \"value\": \"{voltage}\", \"variable\": \"Voltage\"}}, \"PowerActive\": {{\"unit\": \"W\", \"label\": \"\", \"value\": \"{poweractive}\", \"variable\": \"PowerActive\"}}, \"PowerApparent\": {{\"unit\": \"VA\", \"label\": \"\", \"value\": \"{powerapparent}\", \"variable\": \"PowerApparent\"}}, \"PowerReactive\": {{\"unit\": \"VA\", \"label\": \"\", \"value\": \"{powerreactive}\", \"variable\": \"PowerReactive\"}}, \"ExportEnergyActive\": {{\"unit\": \"Wh\", \"label\": \"\", \"value\": \"{exportenergyactive}\", \"variable\": \"ExportEnergyActive\"}}, \"ImportEnergyActive\": {{\"unit\": \"Wh\", \"label\": \"\", \"value\": \"{importenergyactive}\", \"variable\": \"ImportEnergyActive\"}}, \"PowerDemandReverse\": {{\"unit\": \"W\", \"label\": \"\", \"value\": \"{powerdemandreverse}\", \"variable\": \"PowerDemandReverse\"}}, \"PowerDemandPositive\": {{\"unit\": \"W\", \"label\": \"\", \"value\": \"{powerdemandpositive}\", \"variable\": \"PowerDemandPositive\"}}, \"ExportEnergyReactive\": {{\"unit\": \"VAh\", \"label\": \"\", \"value\": \"{exportenergyreactive}\", \"variable\": \"ExportEnergyReactive\"}}, \"ImportEnergyReactive\": {{\"unit\": \"VAh\", \"label\": \"\", \"value\": \"{importenergyreactive}\", \"variable\": \"ImportEnergyReactive\"}}}}"
-            logger.info(f"{destTime=}, {destApplication=}, {destDevice=}, {destAttributes=}, {destValues=}")
-
-
-            try:
-                destCur.execute("insert into measurements (time, application, device, attributes, values) values(%s, %s, %s, %s, %s)",
-                                (destTime, destApplication, destDevice, destAttributes, destValues))
-                destConn.commit()
-            except Exception as e:
-                destConn.rollback()
-                logger.error(f"Error {e} when inserted time {destTime}")
-finally:
-    if srcConn:
-        srcConn.close()
-    if destConn:
-        destConn.close()
-
-

migration/migrate-temperature.py

@@ -1,78 +0,0 @@
-import psycopg2
-from loguru import logger
-import os
-
-srcPgHost = os.environ["SRC_PGHOST"]
-srcPgUser = os.environ["SRC_PGUSER"]
-srcPgPassword = os.environ["SRC_PGPASSWORD"]
-srcPgDatabase = os.environ["SRC_PGDATABASE"]
-destPgHost = os.environ["DEST_PGHOST"]
-destPgUser = os.environ["DEST_PGUSER"]
-destPgPassword = os.environ["DEST_PGPASSWORD"]
-destPgDatabase = os.environ["DEST_PGDATABASE"]
-
-try:
-    srcConn = psycopg2.connect(
-                host=srcPgHost,
-                dbname=srcPgDatabase,
-                user=srcPgUser,
-                password=srcPgPassword,
-                sslmode='require'
-            )
-    srcConn.autocommit = False
-
-    destConn = psycopg2.connect(
-                host=destPgHost,
-                dbname=destPgDatabase,
-                user=destPgUser,
-                password=destPgPassword,
-                sslmode='require'
-            )
-    destConn.autocommit = False
-
-    with srcConn.cursor() as srcCur, destConn.cursor() as destCur:
-        srcCur.execute("select time, location, status, temperature, category from room_climate_measurement_t where category = 'heating' and time > '2023-12-19 05:20:00' order by time")
-        for srcObj in srcCur:
-            timestamp = srcObj[0]
-            location = srcObj[1]
-            status = srcObj[2]
-            temperature = srcObj[3]
-            category = srcObj[4]
-
-            logger.info(f"{timestamp=}, {location=}, {status=}, {temperature=}, {category=}")
-
-            destTime = timestamp
-
-            match category:
-                case 'heating':
-                    destApplication = 'Temperature Heating'
-                case 'Outdoor':
-                    destApplication = 'Temperature Wago'
-                case 'Device':
-                    destApplication = 'Temperature Wago'
-                case 'Indoor':
-                    destApplication = 'Temperature Multisensor' if location != 'Anna-Koeln-2' else 'Temperature Shelly Plus HT'
-                case 'Special':
-                    destApplication = 'Temperature Multisensor'
-
-            destDevice = location
-            destAttributes = '{"ApplicationId":"temperature-imported", "Status":"' + status + '","Location":"' + location + '","Category":"' + category + '","Hint": "Migrated"}'
-            destValues = '{"Value": {"unit": "°C", "label": "", "value": "' + str(temperature) + '", "variable": ""}}'
-
-            logger.info(f"{destTime=}, {destApplication=}, {destDevice=}, {destAttributes=}, {destValues=}")
-
-            try:
-                destCur.execute("insert into measurements (time, application, device, attributes, values) values(%s, %s, %s, %s, %s)",
-                                (destTime, destApplication, destDevice, destAttributes, destValues))
-                destConn.commit()
-            except Exception as e:
-                destConn.rollback()
-                logger.error(f"Error {e} when inserted time {destTime}")
-
-finally:
-    if srcConn:
-        srcConn.close()
-
-    if destConn:
-        destConn.close()
-

migration/requirements.txt

@@ -1,2 +0,0 @@
-loguru==0.7.2
-psycopg2==2.9.9

queries/hottis.sql

@@ -129,3 +129,20 @@ create or replace view lora_sht21_v as
     where m.application = 'de-hottis-app01' and 
           m.attributes->>'DeviceType' = 'hottis-gy21' and
           m.device = d.label;
+
+create or replace view ntp_server_snmp_v as
+  select time,
+         device,
+         cast(values->'load1'->>'value'  as float) as laLoad1,
+         cast(values->'lan-in'->>'value'  as int) as lanInOctetsPerSeconds,
+         cast(values->'lan-out'->>'value'  as int) as lanOutOctetsPerSeconds
+    from measurements
+    where application = 'SNMP' and device = '172.16.13.10';
+
+create or replace view ntp_server_variables_v as
+  select time,
+         device,
+         cast(values->'rootdisp'->>'value'  as float) as rootdisp
+    from measurements
+    where application = 'TSM' and device = '172.16.13.10';
+

queries/saerbeck.sql

@@ -41,3 +41,4 @@ create or replace view cubecell_threeway_battery_v as
     from measurements
     where application = 'de-hottis-saerbeck-monitoring' and
     device = 'eui-70b3d57ed0068fa4';
+

src/udi/database/abstract_database.go

@@ -9,6 +9,7 @@ type VariableType struct {
   Variable string `json:"variable"`
   Unit string `json:"unit"`
   Value interface{} `json:"value,omitempty"`
+  Status string `json:"status,omitempty"`
 }
 
 type Measurement struct {

src/udi/dispatcher/dispatcher.go

@@ -17,7 +17,7 @@ import "udi/handlers/sver"
 import "udi/handlers/svej"
 import "udi/handlers/dt1t"
 import "udi/handlers/locative"
-import "udi/handlers/snmp"
+import "udi/handlers/prepared"
 import "udi/handlers/z2m"
 
 
@@ -49,8 +49,8 @@ func InitDispatcher() {
       factory = dt1t.New
     case "Locative":
       factory = locative.New
-    case "SNMP":
+    case "PREP":
-      factory = snmp.New
+      factory = prepared.New
     case "Z2M":
       factory = z2m.New
     default:

src/udi/go.mod

@@ -3,23 +3,23 @@ module udi
 go 1.22.3
 
 require (
-	github.com/eclipse/paho.mqtt.golang v1.4.3
+	github.com/eclipse/paho.mqtt.golang v1.5.0
-	github.com/google/uuid v1.4.0
+	github.com/google/uuid v1.6.0
 	github.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852
-	gorm.io/driver/postgres v1.5.9
+	gorm.io/driver/postgres v1.5.11
-	gorm.io/gorm v1.25.11
+	gorm.io/gorm v1.25.12
 )
 
 require (
-	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/gorilla/websocket v1.5.3 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
-	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
+	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
-	github.com/jackc/pgx/v5 v5.5.5 // indirect
+	github.com/jackc/pgx/v5 v5.7.2 // indirect
 	github.com/jackc/puddle/v2 v2.2.1 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
-	golang.org/x/crypto v0.25.0 // indirect
+	golang.org/x/crypto v0.32.0 // indirect
-	golang.org/x/net v0.27.0 // indirect
+	golang.org/x/net v0.34.0 // indirect
-	golang.org/x/sync v0.7.0 // indirect
+	golang.org/x/sync v0.10.0 // indirect
-	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
 )

src/udi/go.sum

@@ -28,14 +28,14 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
+golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc=
-golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
+golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
-golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
+golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0=
-golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
+golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
-golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
-golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
-golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

src/udi/handlers/prepared/prepared.go

@@ -1,4 +1,4 @@
-package snmp
+package prepared
 
 import (
   "time"
@@ -10,7 +10,7 @@ import (
 )
 
 
-type SnmpHandler struct {
+type PreparedHandler struct {
   handler.CommonHandler
   dbh *database.DatabaseHandle
 }
@@ -19,6 +19,8 @@ type endpoint_t struct {
   Label string `json:"label"`
   Variable string `json:"variable"`
   Value string `json:"value"`
+  Unit string `json:"unit"`
+  Status string `json:"status"`
 }
 
 type observation_t struct {
@@ -29,16 +31,16 @@ type observation_t struct {
 
 
 func New(id string, config config.HandlerConfigT) handler.Handler {
-  t := &SnmpHandler {
+  t := &PreparedHandler {
   }
   t.Id = id
   t.dbh = database.NewDatabaseHandle()
-  log.Printf("Handler SNMP %d initialized", id)
+  log.Printf("Handler Prepared %d initialized", id)
   return t
 }
 
-func (self *SnmpHandler) Handle(message handler.MessageT) {
+func (self *PreparedHandler) Handle(message handler.MessageT) {
-  // log.Printf("Handler SNMP %d processing %s -> %s", self.Id, message.Topic, message.Payload)
+  log.Printf("Handler Prepared %d processing %s -> %s", self.Id, message.Topic, message.Payload)
 
   var observation observation_t
   err := json.Unmarshal([]byte(message.Payload), &observation)
@@ -50,7 +52,7 @@ func (self *SnmpHandler) Handle(message handler.MessageT) {
   var measurement database.Measurement
   measurement.Time = time.Now()
 
-  measurement.Application = "SNMP"
+  measurement.Application = self.Id
   measurement.Device = observation.Device
 
   measurement.Attributes = map[string]interface{} {
@@ -62,12 +64,13 @@ func (self *SnmpHandler) Handle(message handler.MessageT) {
     measurement.Values[k] = database.VariableType {
       Label: v.Label,
       Variable: v.Variable,
-      Unit: "",
+      Unit: v.Unit,
       Value: v.Value,
+      Status: v.Status,
     }
   }
 
-  // log.Printf("Prepared measurement item: %s", measurement)
+  log.Printf("Prepared measurement item: %s", measurement)
 
   self.dbh.StoreMeasurement(&measurement)
   self.S()