change db password approach

deployment/deploy.sh

@@ -35,9 +35,12 @@ for NAMESPACE_DIR in `find $INSTANCES_DIR -type d -mindepth 1 -maxdepth 1`; do
     INSTANCE=`basename $INSTANCE_DIR`
     echo "Instance: $INSTANCE"
 
+    # set secret configuration from encrypted and decrypted file
+    VARIABLE_PREFIX=`echo "$NAMESPACE""_""$INSTANCE" | tr - _`
+
+    
     # set MQTT_PASSWORD as secret
-    MQTT_PASSWORD_VARIABLE="$NAMESPACE""_""$INSTANCE""_MQTT_PASSWORD"
+    MQTT_PASSWORD_VARIABLE=$VARIABLE_PREFIX"_MQTT_PASSWORD"
-    MQTT_PASSWORD_VARIABLE=`echo $MQTT_PASSWORD_VARIABLE | tr - _`
     MQTT_PASSWORD="${!MQTT_PASSWORD_VARIABLE}"
     # echo "MQTT_PASSWORD_VARIABLE: $MQTT_PASSWORD_VARIABLE"
     # echo "MQTT_PASSWORD: $MQTT_PASSWORD"
@@ -48,25 +51,15 @@ for NAMESPACE_DIR in `find $INSTANCES_DIR -type d -mindepth 1 -maxdepth 1`; do
       --save-config | \
       kubectl apply -f - -n $NAMESPACE
 
-    # set database configuration as secret
-    ## prepare configuration to access database to set udi database password
-    PGUSER=`kubectl get secret -n database timescaledb -o jsonpath="{.data.superuser-username}" | base64 -d`
-    PGHOST=`kubectl get services traefik -n system -o jsonpath="{.status.loadBalancer.ingress[0].ip}"`
-    PGPASSWORD=`kubectl get secret -n database timescaledb -o jsonpath="{.data.superuser-password}" | base64 -d`
-    PGSSLMODE=require
 
-    NEW_UDI_DB_LOGIN="udi""-""$NAMESPACE""-""$INSTANCE"
+    LOGIN_VARIABLE=$VARIABLE_PREFIX"_PGUSER"
-    NEW_UDI_DB_PASSWORD=`tr -dc 'a-zA-Z0-9' < /dev/urandom | head -c 32`
+    NEW_UDI_DB_LOGIN="${!LOGIN_VARIABLE}"
-    NEW_UDI_DB_DATABASE="udi""-""$NAMESPACE""-""$INSTANCE"
+    PASSWORD_VARIABLE=$VARIABLE_PREFIX"_PGDATABASE"
+    NEW_UDI_DB_PASSWORD="${!PASSWORD_VARIABLE}"
+    DATABASE_VARIABLE=$VARIABLE_PREFIX"_PGDATABASE"
+    NEW_UDI_DB_DATABASE="${!DATABASE_VARIABLE}"
     NEW_UDI_DB_HOST=timescaledb.database.svc.cluster.local
 
-    DATABASE_MASTER_POD=`kubectl get pods -n database -l app=StackGresCluster -l role=master -o jsonpath='{.items[0].metadata.name}'`
-    kubectl exec -i $DATABASE_MASTER_POD -c postgres-util -n database -- psql <<EOF
-    BEGIN;
-    ALTER USER "$NEW_UDI_DB_LOGIN" WITH PASSWORD '$NEW_UDI_DB_PASSWORD';
-    COMMIT;
-EOF
-
     kubectl create secret generic $INSTANCE-udi-db-cred \
       --dry-run=client \
       -o yaml \

deployment/instances/udi-saerbeck/level/config.json

@@ -11,13 +11,12 @@
       "handler": "TTN",
       "id": "TTN0",
       "config": {
-        "databaseConnStr": "",
         "attributes": {
         }
       }
     }
   ],
   "archiver": {
-    "dir": "./tmp/udi"
+    "dir": "/archive"
   }
 }

deployment/instances/udi-saerbeck/soil/config.json

@@ -1,17 +1,16 @@
 {
   "mqtt": {
     "broker": "ssl://eu1.cloud.thethings.network:8883",
-    "username": "com-passavant-geiger-poc@ttn",
+    "username": "de-hottis-lora-test1@ttn",
     "password": "ENV",
     "tlsEnable": "true"
   },
   "topicMappings": [
     {
-      "topics": [ "v3/com-passavant-geiger-poc@ttn/devices/#" ],
+      "topics": [ "v3/#" ],
       "handler": "TTN",
       "id": "TTN0",
       "config": {
-        "databaseConnStr": "",
         "attributes": {
         }
       }

deployment/instances/udi/soil/config.json

@@ -1,23 +1,22 @@
 {
   "mqtt": {
     "broker": "ssl://eu1.cloud.thethings.network:8883",
-    "username": "com-passavant-geiger-poc@ttn",
+    "username": "de-hottis-app01@ttn",
     "password": "ENV",
     "tlsEnable": "true"
   },
   "topicMappings": [
     {
-      "topics": [ "v3/com-passavant-geiger-poc@ttn/devices/#" ],
+      "topics": [ "v3/#" ],
       "handler": "TTN",
       "id": "TTN0",
       "config": {
-        "databaseConnStr": "",
         "attributes": {
         }
       }
     }
   ],
   "archiver": {
-    "dir": "./tmp/udi"
+    "dir": "/archive"
   }
 }

deployment/secrets.enc

@@ -1,7 +1,34 @@
-U2FsdGVkX1+jsPd67Sl8bXkH/OVKx3clFUnpJBzwRXA9qEhyDrwMJWg3UH6eiCZd
+U2FsdGVkX1+v9CRp5LutINfnhA50Nwr9JvSomaQcLKASxmGuqlww0UvBr95u3Ton
-PIwApBbDKxR0W/NotaE9100gQo+L3tHL0elMo/YOFkiwkM3tJ1v3zngY1BUoEAGJ
+VFWFT+AlVKftqMXoXcWqcSDZ9dSc2iyBKJglxP1gfY6r+Rc5/FMMYt+blYexv8s3
-4aZNVC7Eb9HDG0RKqh7fSPuazeKxd1fih9nQuxy//XgRivqjbgqfD/xLi5NZ/n0F
+hXmfnzXBFJJsaFAIHaENJacmU2iI89A534WbCrt0yiq3ul4CznuhxCGCFcfUMZPh
-nJT1MsBKGmYF4ezC7thSYIN/+GDL6gpty/FYxAi56oFy6q2YGuADXPgRs1v22vz3
+/6a7wrZ9tIZmt87Z7lxjC765anBoCuoX3iGHw7AtSsBh38SnTARnhcuxEu3aSXJD
-KP+9mIeg6VZn6IRv00w0HSJ3UghMMx/b3pDjIVrq9I3x9sXDD1YqYbjjcnopqKsl
+s+kGRzEmmFDD2vxW95MXkXkDceuzru4oPgWKiTvlvuZS5hER8JVLVZqGYVhsk/+Y
-+gVrX/jCsDBmArRXFu1innOJ+pOsHJ3sSR5fRugnn0Fgje63n4mN46eJYdwhHgEr
+hZmHPn5wVGBbm/3/mPDm0rTnIVOhdXUqnEPGxz0m7h/0i6QvaKo9ylWscxIMoKp4
-yIuyMipc0Uc1ZOsEpkrmYnEofgGaWC3IrdsVEwybbGQ=
+IUKLqEPkULG8OnKzzkCZ3yfyHqUJNFq0KZA0FQXkCifkURGKo9EFz9rYrepGsL7C
+A7TYBL3f5jHQAOmCob2tBp46sh2E4n8RhglAX6JynCdszd65U+DdwERloNlekg+I
++x2Hx94ROYlvYELK1bdITATpijzCsHinfNJ3PHDz85SYh9JAA3ljdOUazoQaT8+n
+R674SnVX7CR793KCgV0wulyHOkAvjkC26mCt4NaEsGW499vYSyzfpneJSx2WY/I2
+MrMNW7IUGjHDZEdSQisGbs3WZd9sZdZFC858QTLKsXVy90lH41ih6SysxNXMSP6G
+Hz7IHGAk2GhzuEj2NALE98hQ9I76T6POOhmREpTlJEkyHGujsW8N4+n5tOx1S2pR
+4Sm8Sl0zQYMHgJcGq/e+HTkCuQ6cuO0yAn2T1oJjwe2OA5OpReljGp6b/DjCY/P6
+/J7hL8EpHa2OTUccSYWNNTfB8oz5oyOnvQTKc3GY6uNJYFfzNkLz9ZsxTi/t6Q3P
+WyxIVCR8dE6z3xPgZxq34K1W44TGsLI/r3DDgTMIhaXr1fcKO69AcAAXFehMqy9L
+Rygg8jQkOxTonSwHQA6MB8oVXfMjsTxKX7NnDHj6dndNRuUMCf5ulmfFCUg7jqxH
+rUL3XOb71mgUWRaI1NY4wOf6wM7NwsGDz+Uyp8LWC7XrOXJsWv6vPNQl/5W4nIRA
+jPYHRf1kC0ZmylB+R5CXh+kf2248iiBPS4fNdEWdj+95hCJ8eSaxu7ROtUt6ZtXO
+aN/0Y9UcU8isjXe7PJilqN8O0KCHHUk4lFlROD/FOkhiPfwoVBfJbObv+TWMB0ir
+pIxeklB1QQrxcxT/ozBi00Bp0Um03nJWU18WRY3lheO+ulEmb9HDvy/iedh2gGVG
+F7JVS68k+5Y8xi+MtcKY3Gm/bh89HjwNgJY7eH+SItpNGoZybbjJNawsd4HnQVS4
+nrtma0RYl3+y48Yf9pWPggKuZ0l7Glji4+dXasDVz7MKOhU9WotIFjmhyLhLkSa+
+tQvEqunAYK8lAXCB6Pgjb7V0AAroLSRR5FGcWaFWXkVYJPBqTopz16W1BgyTH2Bv
+EfYCGSJJXKwuo3jRgHWn/NM3WA25hespcDieKZKU6t50RHiExgySdI4daAM4Kj7X
+OOAIvH1mwOrPGAOpQGOBEaQbUfKcVl3rF0uHAvSy6UeEOFd9NUnb0FSCFcz79SNn
+Q0PkkvII3m9qvRFsKlf6OR6mm5IKARs/juzZxu8zHahJC3NbJ7NcTPMu9LGGrV3t
+vIifvLAsSMhqY2BC+FCVbEIErJa5UhOE6xFOUOi4RDpi4/vfBdCWRoijHQs6HSrk
+c/ihUIOxK580QqHjT4tY9UnwqEY/UlA44SInrSy+ZretWlzls0fLaPbhRv9VNi+v
+sJjAVcwE+56/lPN95599dB3/2PmdzC00BZBhZDQ9v5g/NzMJ/42PO4OWflL9R7Ix
+DVkV8kIuHlU8r+bMavAKDkURGaB/BnsZHLG2754WomcX7WElJYRK7CzlCKvTBGc7
+WvG7QdfUvQj+m0zZHVxf9K1ZK5nw31yM1Y2fOp91oNpgqYv01URpJYs9Ef25L0Nu
+Rj0Unpohunw2LQLp3k/zbSlTny/HmndcOflB4TNvV4nICW+hiU6eH7f/Xy3E+M2d
+2yOBnTfEJlJcLO/Q8ImH7Ex7b30Q+3elkSUlHCU0zTVVYmh3yQIouDvPAjy6Hy7v
+52Mwww/oSasugEQaUpkEFNfs3N0O/s1ngaErqAZtv+5Ab2FwftYTJ+q4lQY8yqRl

migration/.gitignore

@@ -0,0 +1 @@
+.venv

migration/migrate-level.py

@@ -0,0 +1,29 @@
+import psycopg2
+from loguru import logger
+
+try:
+    srcConn = psycopg2.connect(database="level_monitoring_berresheim")
+    srcConn.autocommit = False
+
+    with srcConn.cursor() as srcCur:
+        srcCur.execute("select time, application_name, raw_level, level, status, battery from measurement_t")
+        for srcObj in srcCur:
+            timestamp = srcObj[0]
+            deviceName = srcObj[1]
+            rawLevel = srcObj[2]
+            level = srcObj[3]
+            status = srcObj[4]
+            battery = srcObj[5]
+
+            logger.info(f"{timestamp=}, {deviceName=}, {rawLevel=}, {level=}, {status=}, {battery=}")
+
+            destTime = timestamp
+            destApplication = "de-hottis-level-monitoring"
+            destDevice = "eui-a84041a2c18341d6"
+            destAttributes = '{"ApplicationId":"de-hottis-level-monitoring", "DeviceType":"dragino-ldds75", "Hint": "Migrated"}'
+            destValues = '{"Battery":{"unit":"V","label":"Battery","value":' + str(battery) + ',"variable":"Voltage"}, "Distance":{"unit":mm","label":"Distance","variable":"Level","value":' + str(rawLevel) + '}, "CorrectedDistance":{"unit":"mm", "label":"CorrectedDistance", "variable":"Level","value":' + str(level) + '}}'
+            logger.info(f"{destTime=}, {destApplication=}, {destDevice=}, {destAttributes=}, {destValues=}")
+finally:
+    if srcConn:
+        srcConn.close()
+

migration/requirements.txt

@@ -0,0 +1,2 @@
+loguru==0.7.2
+psycopg2==2.9.9

src/udi/config-iot2.json

@@ -1,21 +0,0 @@
-{
-  "mqtt": {
-    "broker": "mqtt://172.23.1.102:1883",
-    "tlsEnable": "false"
-  },
-  "topicMappings": [
-    {
-      "topics": [ "ttn/#" ],
-      "handler": "TTN",
-      "id": "TTN0",
-      "config": {
-        "databaseConnStr": "",
-        "attributes": {
-        }
-      }
-    }
-  ],
-  "archiver": {
-    "dir": "./tmp/udi"
-  }
-}

src/udi/config-test-ttn.json

@@ -1,8 +1,7 @@
 {
   "mqtt": {
     "broker": "ssl://eu1.cloud.thethings.network:8883",
-    "username": "de-hottis-level-monitoring@ttn",
+    "username": "de-hottis-lora-test1@ttn",
-    "passwordEnvVar": "MQTT_PASSWORD",
     "tlsEnable": "true"
   },
   "topicMappings": [
@@ -11,7 +10,6 @@
       "handler": "TTN",
       "id": "TTN0",
       "config": {
-        "databaseConnStr": "",
         "attributes": {
         }
       }

src/udi/config-ttn.json

@@ -1,29 +0,0 @@
-{
-  "mqtt": {
-    "broker": "mqtt://172.23.1.102:1883",
-    "tlsEnable": "false"
-  },
-  "topicMappings": [
-    {
-      "topics": [ "v3/de-hottis-level-monitoring@ttn/devices/+/up" ],
-      "handler": "TTN"
-    }
-  ],
-  "handlers": [
-    {
-      "name": "TTN",
-      "databaseConnStr": "",
-      "attributes": {
-      }
-    },
-		{
-			"name": "IoT",
-			"databaseConnStr": "",
-			"attributes": {
-			}
-		}
-  ],
-  "archiver": {
-    "dir": "./tmp/udi"
-  }
-}

src/udi/handlers/ttn/models/draginoLse01/draginoLse01.go

@@ -0,0 +1,63 @@
+package draginoLse01
+
+import (
+  "fmt"
+  "encoding/json"
+  "udi/database"
+)
+
+/*
+{
+  "Bat":3.211,
+  "TempC_DS18B20":"0.0",
+  "conduct_SOIL":32,
+  "temp_SOIL":"7.56",
+  "water_SOIL":"25.92"
+}
+*/
+
+type message struct {
+  Bat float32 `json:"Bat"`
+  TempC_DS18B20 string `json:"TempC_DS18B20"`
+  Conduct_SOIL int `json:"conduct_SOIL"`
+  Temp_SOIL string `json:"temp_SOIL"`
+  Water_SOIL string `json:"water_SOIL"`
+}
+
+func Parse(fPort int, decodedPayload []byte, variables *map[string]database.VariableType, device *database.Device) error {
+  if fPort != 2 {
+    return fmt.Errorf("Unexpected fPort %d", fPort)
+  }
+  var message message
+  err := json.Unmarshal(decodedPayload, &message)
+  if err != nil {
+    return fmt.Errorf("Unable to parse payload, fPort %d, error %s", fPort, err)
+  }
+  (*variables)["Battery"] = database.VariableType {
+    Label: "Battery",
+    Variable: "Voltage",
+    Unit: "V",
+    Value: message.Bat,
+  }
+  (*variables)["Conductance"] = database.VariableType {
+    Label: "Conductance",
+    Variable: "Conductance",
+    Unit: "uS/cm",
+    Value: message.Conduct_SOIL,
+  }
+  (*variables)["Temperature"] = database.VariableType {
+    Label: "Temperature",
+    Variable: "Temperature",
+    Unit: "°C",
+    Value: message.Temp_SOIL,
+  }
+  (*variables)["Water"] = database.VariableType {
+    Label: "Water",
+    Variable: "Water",
+    Unit: "%",
+    Value: message.Water_SOIL,
+  }
+
+  return nil
+}
+

src/udi/handlers/ttn/ttn.go

@@ -2,6 +2,7 @@ package ttn
 
 import (
   "fmt"
+  "log"
   "time"
   "encoding/json"
   "udi/config"
@@ -9,6 +10,7 @@ import (
   "udi/handlers/ttn/models/emuProfIILoRaCfg1"
   "udi/handlers/ttn/models/draginoLdds75"
   "udi/handlers/ttn/models/draginoLmds200"
+  "udi/handlers/ttn/models/draginoLse01"
   "udi/database"
 )
 
@@ -83,7 +85,7 @@ func New(id string, config config.HandlerConfigT) handler.Handler {
 }
 
 func (self *TTNHandler) Handle(message handler.MessageT) {
-  // log.Printf("Handler TTN %d processing %s -> %s", self.id, message.Topic, message.Payload)
+  log.Printf("Handler TTN %d processing %s -> %s", self.Id, message.Topic, message.Payload)
 
   var measurement database.Measurement
   measurement.Time = time.Now()
@@ -94,7 +96,7 @@ func (self *TTNHandler) Handle(message handler.MessageT) {
     self.Lost("Error when unmarshaling message", err, message)
     return
   }
-  //log.Printf("Parsed message: %s", uplinkMessage)
+  log.Printf("Parsed message: %s", uplinkMessage)
 
   var attributes attributes
   attributes.DeviceId = uplinkMessage.EndDeviceIds.DeviceId
@@ -104,11 +106,11 @@ func (self *TTNHandler) Handle(message handler.MessageT) {
   attributes.FrmPayload = uplinkMessage.UplinkMessage.FrmPayload
   attributes.ConsumedAirtime = uplinkMessage.UplinkMessage.ConsumedAirtime
   for _, rxm := range uplinkMessage.UplinkMessage.RxMetadata {
-    //log.Printf("RXM: %s", rxm)
+    log.Printf("RXM: %s", rxm)
     g := gatewayAttributes { GatewayId: rxm.GatewayIds.GatewayId, Rssi: rxm.Rssi, Snr: rxm.Snr }
     attributes.Gateways = append(attributes.Gateways, g)
   }
-  //log.Printf("Attributes: %s", attributes)
+  log.Printf("Attributes: %s", attributes)
   measurement.Attributes = map[string]interface{} {
     "DeviceId": attributes.DeviceId,
     "ApplicationId": attributes.ApplicationId,
@@ -119,7 +121,7 @@ func (self *TTNHandler) Handle(message handler.MessageT) {
     "ConsumedAirtime": attributes.ConsumedAirtime,
   }
 
-  //log.Printf("ApplicationId: %s, DeviceId: %s", attributes.ApplicationId, attributes.DeviceId)
+  log.Printf("ApplicationId: %s, DeviceId: %s", attributes.ApplicationId, attributes.DeviceId)
   device, err2 := self.dbh.GetDeviceByLabelAndApplication(attributes.ApplicationId, attributes.DeviceId)
   if err2 != nil {
     self.Lost("Error when loading device", err2, message)
@@ -129,7 +131,7 @@ func (self *TTNHandler) Handle(message handler.MessageT) {
   measurement.Device = attributes.DeviceId
   measurement.Attributes["DeviceType"] = device.DeviceType.ModelIdentifier
 
-  //log.Printf("DeviceLabel: %s, DeviceType: %s", device.Label, device.DeviceType.ModelIdentifier)
+  log.Printf("DeviceLabel: %s, DeviceType: %s", device.Label, device.DeviceType.ModelIdentifier)
 
   var parser func(int, []byte, *map[string]database.VariableType, *database.Device) error
   switch device.DeviceType.ModelIdentifier {
@@ -139,6 +141,8 @@ func (self *TTNHandler) Handle(message handler.MessageT) {
     parser = draginoLdds75.Parse
   case "dragino-lmds200":
     parser = draginoLmds200.Parse
+  case "dragino-lse01":
+    parser = draginoLse01.Parse
   default:
     self.Lost(fmt.Sprintf("No parser found for %s", device.DeviceType.ModelIdentifier), nil, message)
     return
@@ -150,7 +154,7 @@ func (self *TTNHandler) Handle(message handler.MessageT) {
     self.Lost("Model parser failed", err3, message)
     return
   }
-  //log.Printf("Prepared measurement item: %s", measurement)
+  log.Printf("Prepared measurement item: %s", measurement)
   self.dbh.StoreMeasurement(&measurement)
   self.S()
 }