wn/universal-data-ingest
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

emu stuff
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
Details

Browse Source
This commit is contained in:
Wolfgang Hottgenroth
2023-12-12 16:39:22 +01:00
parent e99c9023a0
commit 2dd830907d
6 changed files with 350 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/udi/ENVDB.udi Normal file
View File

@ -0,0 +1,6 @@
PGUSER="udi"
PGHOST=`kubectl get services traefik -n system -o jsonpath="{.status.loadBalancer.ingress[0].ip}"`
PGPASSWORD=`kubectl get secrets udi-db-cred -n udi -o jsonpath="{.data.PGPASSWORD}" | base64 --decode`
PGSSLMODE=require
PGDATABASE="uditest"
export PGUSER PGHOST PGPASSWORD PGSSLMODE PGDATABASE

0
src/udi/handlers/ttn/example.json → src/udi/handlers/ttn/lmds200-example.json
View File

124
src/udi/handlers/ttn/models/emuProfIILoRaCfg1/emu-fprot1-example.json Normal file
View File

@ -0,0 +1,124 @@
{
"end_device_ids": {
"device_id": "eui-102ceffffe01089c",
"application_ids": {
"application_id": "com-passavant-geiger-poc"
},
"dev_eui": "102CEFFFFE01089C",
"join_eui": "102CEF0000000000",
"dev_addr": "260B0E1A"
},
"correlation_ids": [
"gs:uplink:01HHF7YF14Y7HQBF9D8N8D20ZM"
],
"received_at": "2023-12-12T14:47:26.197129491Z",
"uplink_message": {
"session_key_id": "AYxJcJyrJgr7XiIUdO3EBA==",
"f_port": 1,
"f_cnt": 11738,
"frm_payload": "7HF4ZQsAAAAADAAAAAANAAAAAA4AAAAAFwAYABkA8ADc",
"decoded_payload": {
"Active Power L1": {
"cfgphase": 1,
"unit": "W",
"value": 0
},
"Active Power L123": {
"unit": "W",
"value": 0
},
"Active Power L2": {
"cfgphase": 2,
"unit": "W",
"value": 0
},
"Active Power L3": {
"cfgphase": 3,
"unit": "W",
"value": 0
},
"Powerfactor L1": {
"cfgphase": 1,
"unit": "Cos",
"value": 0,
"value_raw": 0
},
"Powerfactor L2": {
"cfgphase": 2,
"unit": "Cos",
"value": 0,
"value_raw": 0
},
"Powerfactor L3": {
"cfgphase": 3,
"unit": "Cos",
"value": 0,
"value_raw": 0
},
"errorcode": {
"CTRatioChange": false,
"ImpulseRatioChange": false,
"ImpulseWidthChange": false,
"LogbookFull": false,
"PowerFail": false,
"TimeChanged": false,
"VTRatioChange": false,
"value": 0
},
"medium": {
"desc": "Electricity",
"type": 1
},
"timeStamp": 1702392300
},
"rx_metadata": [
{
"gateway_ids": {
"gateway_id": "eui-b827ebfffe8b01dd",
"eui": "B827EBFFFE8B01DD"
},
"time": "2023-12-12T14:47:25.951668977Z",
"timestamp": 3710351237,
"rssi": -89,
"channel_rssi": -89,
"snr": 14,
"location": {
"latitude": 51.404164272478724,
"longitude": 7.060088589208832,
"source": "SOURCE_REGISTRY"
},
"uplink_token": "CiIKIAoUZXVpLWI4MjdlYmZmZmU4YjAxZGQSCLgn6//+iwHdEIXvnekNGgwI/eThqwYQpoGI1wMgiL+rkf5r",
"received_at": "2023-12-12T14:47:25.969479115Z"
}
],
"settings": {
"data_rate": {
"lora": {
"bandwidth": 125000,
"spreading_factor": 7,
"coding_rate": "4/5"
}
},
"frequency": "867100000",
"timestamp": 3710351237,
"time": "2023-12-12T14:47:25.951668977Z"
},
"received_at": "2023-12-12T14:47:25.988957776Z",
"confirmed": true,
"consumed_airtime": "0.092416s",
"version_ids": {
"brand_id": "emu",
"model_id": "emu-prof-ii",
"hardware_version": "1.0",
"firmware_version": "1.0",
"band_id": "EU_863_870"
},
"network_ids": {
"net_id": "000013",
"ns_id": "EC656E0000000181",
"tenant_id": "ttn",
"cluster_id": "eu1",
"cluster_address": "eu1.cloud.thethings.network"
}
}
}

0
src/udi/handlers/ttn/emu-example.json → src/udi/handlers/ttn/models/emuProfIILoRaCfg1/emu-fprot2-example.json
View File

209
src/udi/handlers/ttn/models/emuProfIILoRaCfg1/emuProfIILoRaCfg1.go
View File

@ -7,14 +7,11 @@ package emuProfIILoRaCfg1
import (
"fmt"
//"log"
"encoding/json"
"udi/database"
)
type emuVariable struct {
Value interface{} `json:"value"`
Unit string `json:"unit"`
}
/*
{
@ -41,46 +38,216 @@ type emuVariable struct {
"timeStamp": 1702052100
}
*/
type emuMessage1 struct {
ActiveEnergyExport emuVariable `json:"Active Energy Export T1 64bit"`
ReactiveEnergyExport emuVariable `json:"Reactive Energy Export T1 64bit"`
ActiveEnergyImport emuVariable `json:"Active Energy Import T1 64bit"`
ReactiveEnergyImport emuVariable `json:"Reactive Energy Import T1 64bit"`
type emuMessage2 struct {
ActiveEnergyExport struct {
Value int `json:"value"`
Unit string `json:"unit"`
} `json:"Active Energy Export T1 64bit"`
ReactiveEnergyExport struct {
Value int `json:"value"`
Unit string `json:"unit"`
} `json:"Reactive Energy Export T1 64bit"`
ActiveEnergyImport struct {
Value int `json:"value"`
Unit string `json:"unit"`
} `json:"Active Energy Import T1 64bit"`
ReactiveEnergyImport struct {
Value int `json:"value"`
Unit string `json:"unit"`
} `json:"Reactive Energy Import T1 64bit"`
Medium struct {
Desc string `json:"desc"`
Type string `json:"type"`
Type int `json:"type"`
} `json:"medium"`
Timestamp int `json:"timestamp"`
}
func Parse(fPort int, decodedPayload map[string]interface{}) (map[string]database.VariableType, error) {
/*
{
"Active Power L1": {
"cfgphase": 1,
"unit": "W",
"value": 0
},
"Active Power L123": {
"unit": "W",
"value": 0
},
"Active Power L2": {
"cfgphase": 2,
"unit": "W",
"value": 0
},
"Active Power L3": {
"cfgphase": 3,
"unit": "W",
"value": 0
},
"Powerfactor L1": {
"cfgphase": 1,
"unit": "Cos",
"value": 0,
"value_raw": 0
},
"Powerfactor L2": {
"cfgphase": 2,
"unit": "Cos",
"value": 0,
"value_raw": 0
},
"Powerfactor L3": {
"cfgphase": 3,
"unit": "Cos",
"value": 0,
"value_raw": 0
},
"errorcode": {
"CTRatioChange": false,
"ImpulseRatioChange": false,
"ImpulseWidthChange": false,
"LogbookFull": false,
"PowerFail": false,
"TimeChanged": false,
"VTRatioChange": false,
"value": 0
},
"medium": {
"desc": "Electricity",
"type": 1
},
"timeStamp": 1702392300
}
*/
type emuMessage1 struct {
ActivePowerL1 struct {
CfgPhase int `json:"cfgphase"`
Unit string `json:"unit"`
Value int `json:"value"`
} `json:"Active Power L1"`
ActivePowerL2 struct {
CfgPhase int `json:"cfgphase"`
Unit string `json:"unit"`
Value int `json:"value"`
} `json:"Active Power L2"`
ActivePowerL3 struct {
CfgPhase int `json:"cfgphase"`
Unit string `json:"unit"`
Value int `json:"value"`
} `json:"Active Power L3"`
ActivePowerL123 struct {
Unit string `json:"unit"`
Value int `json:"value"`
} `json:"Active Power L123"`
PowerfactorL1 struct {
CfgPhase int `json:"cfgphase"`
Unit string `json:"unit"`
Value float32 `json:"value"`
ValueRaw float32 `json:"value_raw"`
} `json:"Powerfactor L1"`
PowerfactorL2 struct {
CfgPhase int `json:"cfgphase"`
Unit string `json:"unit"`
Value float32 `json:"value"`
ValueRaw float32 `json:"value_raw"`
} `json:"Powerfactor L2"`
PowerfactorL3 struct {
CfgPhase int `json:"cfgphase"`
Unit string `json:"unit"`
Value float32 `json:"value"`
ValueRaw float32 `json:"value_raw"`
} `json:"Powerfactor L3"`
ErrorCode struct {
CTRatioChange bool `json:"CTRatioChange"`
ImpulseRatioChange bool `json:"ImpulseRatioChange"`
ImpulseWidthChange bool `json:"ImpulseWidthChange"`
LogbookFull bool `json:"LogbookFull"`
PowerFail bool `json:"PowerFail"`
TimeChanged bool `json:"TimeChanged"`
VTRatioChange bool `json:"VTRatioChange"`
Value int `json:"value"`
} `json:"errorcode"`
Medium struct {
Desc string `json:"desc"`
Type int `json:"type"`
} `json:"medium"`
Timestamp int `json:"timestamp"`
}
func Parse(fPort int, decodedPayload []byte) (map[string]database.VariableType, error) {
//log.Printf("Parse input: %d, %s", fPort, decodedPayload)
switch fPort {
case 1:
var emuMessage1 emuMessage1
err := json.Unmarshal([]byte(decodedPayload), &emuMessage1)
err := json.Unmarshal(decodedPayload, &emuMessage1)
if err != nil {
return nil, fmt.Errorf("Unable to parse payload, fPort %d", fPort)
return nil, fmt.Errorf("Unable to parse payload, fPort %d, error %s", fPort, err)
}
variables := make(map[string]database.VariableType)
variables["ActivePowerL1"] = database.VariableType {
Variable: "ActivePowerL1",
Unit: emuMessage1.ActivePowerL1.Unit,
Value: emuMessage1.ActivePowerL1.Value,
}
variables["ActivePowerL2"] = database.VariableType {
Variable: "ActivePowerL2",
Unit: emuMessage1.ActivePowerL2.Unit,
Value: emuMessage1.ActivePowerL2.Value,
}
variables["ActivePowerL3"] = database.VariableType {
Variable: "ActivePowerL3",
Unit: emuMessage1.ActivePowerL3.Unit,
Value: emuMessage1.ActivePowerL3.Value,
}
variables["ActivePowerL123"] = database.VariableType {
Variable: "ActivePowerL123",
Unit: emuMessage1.ActivePowerL123.Unit,
Value: emuMessage1.ActivePowerL123.Value,
}
variables["PowerfactorL1"] = database.VariableType {
Variable: "PowerfactorL1",
Unit: emuMessage1.PowerfactorL1.Unit,
Value: emuMessage1.PowerfactorL1.Value,
}
variables["PowerfactorL2"] = database.VariableType {
Variable: "PowerfactorL2",
Unit: emuMessage1.PowerfactorL2.Unit,
Value: emuMessage1.PowerfactorL2.Value,
}
variables["PowerfactorL3"] = database.VariableType {
Variable: "PowerfactorL3",
Unit: emuMessage1.PowerfactorL3.Unit,
Value: emuMessage1.PowerfactorL3.Value,
}
return variables, nil
case 2:
var emuMessage2 emuMessage2
err := json.Unmarshal(decodedPayload, &emuMessage2)
if err != nil {
return nil, fmt.Errorf("Unable to parse payload, fPort %d, error %s", fPort, err)
}
variables := make(map[string]database.VariableType)
variables["ActiveEnergyExport"] = database.VariableType {
Variable: "ActiveEnergyExport",
Unit: emuMessage1.ActiveEnergyExport.Unit,
Value: emuMessage1.ActiveEnergyExport.Value,
Unit: emuMessage2.ActiveEnergyExport.Unit,
Value: emuMessage2.ActiveEnergyExport.Value,
}
variables["ActiveEnergyImport"] = database.VariableType {
Variable: "ActiveEnergyImport",
Unit: emuMessage1.ActiveEnergyImport.Unit,
Value: emuMessage1.ActiveEnergyImport.Value,
Unit: emuMessage2.ActiveEnergyImport.Unit,
Value: emuMessage2.ActiveEnergyImport.Value,
}
variables["ReactiveEnergyExport"] = database.VariableType {
Variable: "ReactiveEnergyExport",
Unit: emuMessage1.ReactiveEnergyExport.Unit,
Value: emuMessage1.ReactiveEnergyExport.Value,
Unit: emuMessage2.ReactiveEnergyExport.Unit,
Value: emuMessage2.ReactiveEnergyExport.Value,
}
variables["ReactiveEnergyImport"] = database.VariableType {
Variable: "ReactiveEnergyImport",
Unit: emuMessage1.ReactiveEnergyImport.Unit,
Value: emuMessage1.ReactiveEnergyImport.Value,
Unit: emuMessage2.ReactiveEnergyImport.Unit,
Value: emuMessage2.ReactiveEnergyImport.Value,
}
return variables, nil
default:

40
src/udi/handlers/ttn/ttn.go
View File

@ -2,6 +2,7 @@ package ttn
import "log"
import "fmt"
import "time"
import "encoding/json"
import "udi/config"
import "udi/handlers/handler"
@ -15,6 +16,10 @@ type TTNHandler struct {
dbh *database.DatabaseHandle
}
type DecodedPayloaderHolder struct {
Payload []byte
}
type uplinkMessage struct {
EndDeviceIds struct {
DeviceId string `json:"device_id"`
@ -30,7 +35,7 @@ type uplinkMessage struct {
FCnt int `json:"f_cnt"`
FPort int `json:"f_port"`
FrmPayload string `json:"frm_payload"`
DecodedPayload map[string]interface{} `json:"decoded_payload"`
DecodedPayload DecodedPayloaderHolder `json:"decoded_payload"`
RxMetadata []struct {
GatewayIds struct {
GatewayId string `json:"gateway_id"`
@ -62,7 +67,10 @@ type attributes struct {
ConsumedAirtime string `json:"consumed_airtime"`
}
func (self *DecodedPayloaderHolder) UnmarshalJSON(data []byte) error {
self.Payload = data
return nil
}
func NewTTNHandler(config config.HandlerConfigT) handler.Handler {
t := &TTNHandler {
@ -82,7 +90,10 @@ func lost(msg string, message handler.MessageT) {
}
func (self *TTNHandler) Handle(message handler.MessageT) {
log.Printf("Handler TTN %d processing %s -> %s", self.id, message.Topic, message.Payload)
// log.Printf("Handler TTN %d processing %s -> %s", self.id, message.Topic, message.Payload)
var measurement database.Measurement
measurement.Time = time.Now()
var uplinkMessage uplinkMessage
err := json.Unmarshal([]byte(message.Payload), &uplinkMessage)
@ -105,17 +116,28 @@ func (self *TTNHandler) Handle(message handler.MessageT) {
attributes.Gateways = append(attributes.Gateways, g)
}
//log.Printf("Attributes: %s", attributes)
measurement.Attributes = map[string]interface{} {
"DeviceId": attributes.DeviceId,
"ApplicationId": attributes.ApplicationId,
"FCnt": attributes.FCnt,
"FPort": attributes.FPort,
"FrmPayload": attributes.FrmPayload,
"Gateways": attributes.Gateways,
"ConsumedAirtime": attributes.ConsumedAirtime,
}
log.Printf("ApplicationId: %s, DeviceId: %s", attributes.ApplicationId, attributes.DeviceId)
//log.Printf("ApplicationId: %s, DeviceId: %s", attributes.ApplicationId, attributes.DeviceId)
device, err2 := self.dbh.GetDeviceByLabelAndApplication(attributes.ApplicationId, attributes.DeviceId)
if err2 != nil {
lost(fmt.Sprintf("Error when loading device: %s, ", err2), message)
return
}
measurement.Application = attributes.ApplicationId
measurement.Device = attributes.DeviceId
log.Printf("DeviceLabel: %s, DeviceType: %s", device.Label, device.DeviceType.ModelIdentifier)
//log.Printf("DeviceLabel: %s, DeviceType: %s", device.Label, device.DeviceType.ModelIdentifier)
var parser func(int, interface{}) (map[string]database.VariableType, error)
var parser func(int, []byte) (map[string]database.VariableType, error)
switch device.DeviceType.ModelIdentifier {
case "emu-prof-ii-lora-cfg1":
parser = emuProfIILoRaCfg1.Parse
@ -124,12 +146,14 @@ func (self *TTNHandler) Handle(message handler.MessageT) {
return
}
_, err3 := parser(uplinkMessage.UplinkMessage.FPort, uplinkMessage.UplinkMessage.DecodedPayload)
variables, err3 := parser(uplinkMessage.UplinkMessage.FPort, uplinkMessage.UplinkMessage.DecodedPayload.Payload)
if err3 != nil {
lost(fmt.Sprintf("Model parser failed: %s", err3), message)
return
}
measurement.Values = variables
log.Printf("Prepared measurement item: %s", measurement)
self.dbh.StoreMeasurement(&measurement)
}