From c54b335e5f418a02b1ffcdbf67525b6a8b4d0a23 Mon Sep 17 00:00:00 2001 From: Wolfgang Hottgenroth Date: Wed, 3 Jan 2024 20:57:26 +0100 Subject: [PATCH] queries and migrations --- migration/migrate-pv.py | 79 +++++++++++++++++++++++++ migration/migrate-temperature.py | 78 ++++++++++++++++++++++++ queries/berresheim.sql | 8 +++ queries/hottis.sql | 15 +++++ queries/old-daily-temperature-query.sql | 11 ++++ queries/pg.sql | 8 +-- src/udi/ENVDB.udiload | 12 ++-- 7 files changed, 200 insertions(+), 11 deletions(-) create mode 100644 migration/migrate-pv.py create mode 100644 migration/migrate-temperature.py create mode 100644 queries/berresheim.sql create mode 100644 queries/old-daily-temperature-query.sql diff --git a/migration/migrate-pv.py b/migration/migrate-pv.py new file mode 100644 index 0000000..0a924f6 --- /dev/null +++ b/migration/migrate-pv.py @@ -0,0 +1,79 @@ +import psycopg2 +from loguru import logger +import os + +srcPgHost = os.environ["SRC_PGHOST"] +srcPgUser = os.environ["SRC_PGUSER"] +srcPgPassword = os.environ["SRC_PGPASSWORD"] +srcPgDatabase = os.environ["SRC_PGDATABASE"] +destPgHost = os.environ["DEST_PGHOST"] +destPgUser = os.environ["DEST_PGUSER"] +destPgPassword = os.environ["DEST_PGPASSWORD"] +destPgDatabase = os.environ["DEST_PGDATABASE"] + +try: + srcConn = psycopg2.connect( + host=srcPgHost, + dbname=srcPgDatabase, + user=srcPgUser, + password=srcPgPassword, + sslmode='require' + ) + srcConn.autocommit = False + + destConn = psycopg2.connect( + host=destPgHost, + dbname=destPgDatabase, + user=destPgUser, + password=destPgPassword, + sslmode='require' + ) + destConn.autocommit = False + + with srcConn.cursor() as srcCur, destConn.cursor() as destCur: + srcCur.execute("select time, deviceid, status, state, importenergyactive, importenergyreactive, exportenergyactive, exportenergyreactive, powerapparent, poweractive, powerreactive, powerdemandpositive, powerdemandreverse, factor, angle, voltage, current, powerdemand from pv_power_measurement order by time") + for srcObj in srcCur: + timestamp = srcObj[0] + deviceName = srcObj[1] + status = srcObj[2] + state = srcObj[3] + importenergyactive = srcObj[4] + importenergyreactive = srcObj[5] + exportenergyactive = srcObj[6] + exportenergyreactive = srcObj[7] + powerapparent = srcObj[8] + poweractive = srcObj[9] + powerreactive = srcObj[10] + powerdemandpositive = srcObj[11] + powerdemandreverse = srcObj[12] + factor = srcObj[13] + angle = srcObj[14] + voltage = srcObj[15] + current = srcObj[16] + powerdemand = srcObj[17] + + + logger.info(f"{timestamp=}, {deviceName=}") + + destTime = timestamp + destApplication = "PV" + destDevice = "Powermeter" + destAttributes = f"{\"ApplicationId\":\"PV\", \"Status\":\"{status}\",\"Hint\": \"Migrated\"}' + destValues = f"{\"Cnt\": {\"unit\": \"\", \"label\": \"\", \"value\": \"-1\", \"variable\": \"Cnt\"}, \"Angle\": {\"unit\": \"degree\", \"label\": \"\", \"value\": \"{angle}\", \"variable\": \"Angle\"}, \"State\": {\"unit\": \"\", \"label\": \"\", \"value\": \"{state}\", \"variable\": \"State\"}, \"Factor\": {\"unit\": \"\", \"label\": \"\", \"value\": \"{factor}\", \"variable\": \"Factor\"}, \"Current\": {\"unit\": \"A\", \"label\": \"\", \"value\": \"{current}\", \"variable\": \"Current\"}, \"Voltage\": {\"unit\": \"V\", \"label\": \"\", \"value\": \"{voltage}\", \"variable\": \"Voltage\"}, \"PowerActive\": {\"unit\": \"W\", \"label\": \"\", \"value\": \"{poweractive}\", \"variable\": \"PowerActive\"}, \"PowerApparent\": {\"unit\": \"VA\", \"label\": \"\", \"value\": \"{powerapparent}\", \"variable\": \"PowerApparent\"}, \"PowerReactive\": {\"unit\": \"VA\", \"label\": \"\", \"value\": \"{powerreactive}\", \"variable\": \"PowerReactive\"}, \"ExportEnergyActive\": {\"unit\": \"Wh\", \"label\": \"\", \"value\": \"{exportenergyactive}\", \"variable\": \"ExportEnergyActive\"}, \"ImportEnergyActive\": {\"unit\": \"Wh\", \"label\": \"\", \"value\": \"{importenergyactive}\", \"variable\": \"ImportEnergyActive\"}, \"PowerDemandReverse\": {\"unit\": \"W\", \"label\": \"\", \"value\": \"{powerdemandreverse}\", \"variable\": \"PowerDemandReverse\"}, \"PowerDemandPositive\": {\"unit\": \"W\", \"label\": \"\", \"value\": \"{powerdemandpositive}\", \"variable\": \"PowerDemandPositive\"}, \"ExportEnergyReactive\": {\"unit\": \"VAh\", \"label\": \"\", \"value\": \"{exportenergyreactive}\", \"variable\": \"ExportEnergyReactive\"}, \"ImportEnergyReactive\": {\"unit\": \"VAh\", \"label\": \"\", \"value\": \"{importenergyreactive}\", \"variable\": \"ImportEnergyReactive\"}}" + logger.info(f"{destTime=}, {destApplication=}, {destDevice=}, {destAttributes=}, {destValues=}") + + + try: + destCur.execute("insert into measurements (time, application, device, attributes, values) values(%s, %s, %s, %s, %s)", + (destTime, destApplication, destDevice, destAttributes, destValues)) + destConn.commit() + except Exception as e: + destConn.rollback() + logger.error(f"Error {e} when inserted time {destTime}") +finally: + if srcConn: + srcConn.close() + if destConn: + destConn.close() + + diff --git a/migration/migrate-temperature.py b/migration/migrate-temperature.py new file mode 100644 index 0000000..707ee46 --- /dev/null +++ b/migration/migrate-temperature.py @@ -0,0 +1,78 @@ +import psycopg2 +from loguru import logger +import os + +srcPgHost = os.environ["SRC_PGHOST"] +srcPgUser = os.environ["SRC_PGUSER"] +srcPgPassword = os.environ["SRC_PGPASSWORD"] +srcPgDatabase = os.environ["SRC_PGDATABASE"] +destPgHost = os.environ["DEST_PGHOST"] +destPgUser = os.environ["DEST_PGUSER"] +destPgPassword = os.environ["DEST_PGPASSWORD"] +destPgDatabase = os.environ["DEST_PGDATABASE"] + +try: + srcConn = psycopg2.connect( + host=srcPgHost, + dbname=srcPgDatabase, + user=srcPgUser, + password=srcPgPassword, + sslmode='require' + ) + srcConn.autocommit = False + + destConn = psycopg2.connect( + host=destPgHost, + dbname=destPgDatabase, + user=destPgUser, + password=destPgPassword, + sslmode='require' + ) + destConn.autocommit = False + + with srcConn.cursor() as srcCur, destConn.cursor() as destCur: + srcCur.execute("select time, location, status, temperature, category from room_climate_measurement_t where category = 'heating' and time > '2023-12-19 05:20:00' order by time") + for srcObj in srcCur: + timestamp = srcObj[0] + location = srcObj[1] + status = srcObj[2] + temperature = srcObj[3] + category = srcObj[4] + + logger.info(f"{timestamp=}, {location=}, {status=}, {temperature=}, {category=}") + + destTime = timestamp + + match category: + case 'heating': + destApplication = 'Temperature Heating' + case 'Outdoor': + destApplication = 'Temperature Wago' + case 'Device': + destApplication = 'Temperature Wago' + case 'Indoor': + destApplication = 'Temperature Multisensor' if location != 'Anna-Koeln-2' else 'Temperature Shelly Plus HT' + case 'Special': + destApplication = 'Temperature Multisensor' + + destDevice = location + destAttributes = '{"ApplicationId":"temperature-imported", "Status":"' + status + '","Location":"' + location + '","Category":"' + category + '","Hint": "Migrated"}' + destValues = '{"Value": {"unit": "°C", "label": "", "value": "' + str(temperature) + '", "variable": ""}}' + + logger.info(f"{destTime=}, {destApplication=}, {destDevice=}, {destAttributes=}, {destValues=}") + + try: + destCur.execute("insert into measurements (time, application, device, attributes, values) values(%s, %s, %s, %s, %s)", + (destTime, destApplication, destDevice, destAttributes, destValues)) + destConn.commit() + except Exception as e: + destConn.rollback() + logger.error(f"Error {e} when inserted time {destTime}") + +finally: + if srcConn: + srcConn.close() + + if destConn: + destConn.close() + diff --git a/queries/berresheim.sql b/queries/berresheim.sql new file mode 100644 index 0000000..38c06f3 --- /dev/null +++ b/queries/berresheim.sql @@ -0,0 +1,8 @@ +create or replace view level_v as + select time, + cast(values->'CorrectedDistance'->>'value' as float) as level, + cast(values->'Battery'->>'value' as float) as battery, + attributes->>'Status' as status, + device + from measurements + where application = 'de-hottis-level-monitoring'; diff --git a/queries/hottis.sql b/queries/hottis.sql index b06e60d..9f2c6f5 100644 --- a/queries/hottis.sql +++ b/queries/hottis.sql @@ -45,6 +45,13 @@ create or replace view temperature_v as from measurements where application in ('Temperature Multisensor', 'Temperature Shelly Plus HT'); +create or replace view temperature2_v as + select time, + cast(values->'Value'->>'value' as float) as temperature, + device + from measurements + where application = 'Temperature Wago'; + create or replace view humidity_v as select time, cast(values->'Value'->>'value' as float) as humidity, @@ -52,3 +59,11 @@ create or replace view humidity_v as from measurements where application in ('Humidity Multisensor'); +create or replace view soil_v as + select time, + cast(values->'Water'->>'value' as float) as water, + cast(values->'Conductance'->>'value' as float) as conductance, + cast(values->'Temperature'->>'value' as float) as temperature, + device + from measurements + where application = 'de-hottis-app01' and attributes->>'DeviceType' = 'dragino-lse01'; diff --git a/queries/old-daily-temperature-query.sql b/queries/old-daily-temperature-query.sql new file mode 100644 index 0000000..49fdad6 --- /dev/null +++ b/queries/old-daily-temperature-query.sql @@ -0,0 +1,11 @@ +select + extract('day' from time)::varchar || '.' || extract('month' from time)::varchar || '.' || extract('year' from time)::varchar as day, + avg(temperature)::numeric(10,0) as temperature + from room_climate_measurement_t + where + category = 'Outdoor' and + location = 'Outdoor' and + extract('hour' from time) = 12 and + time::date = now()::date + group by day + diff --git a/queries/pg.sql b/queries/pg.sql index 21d7f41..ab97904 100644 --- a/queries/pg.sql +++ b/queries/pg.sql @@ -3,13 +3,7 @@ create or replace view power_v as cast(values->'ActivePowerL1'->>'value' as float) as power_l1, cast(values->'ActivePowerL2'->>'value' as float) as power_l2, cast(values->'ActivePowerL3'->>'value' as float) as power_l3, - device - from measurements - where application = 'com-passavant-geiger-poc' and - attributes->>'FPort' = '1'; - -create or replace view power_factor_v as - select time, + cast(values->'ActivePowerL123'->>'value' as float) as power_total, cast(values->'PowerfactorL1'->>'value' as float) as factor_l1, cast(values->'PowerfactorL2'->>'value' as float) as factor_l2, cast(values->'PowerfactorL3'->>'value' as float) as factor_l3, diff --git a/src/udi/ENVDB.udiload b/src/udi/ENVDB.udiload index 0d0820b..8ddfcb0 100644 --- a/src/udi/ENVDB.udiload +++ b/src/udi/ENVDB.udiload @@ -2,9 +2,13 @@ if [ "$1" = "" ]; then echo "set namespace as argument" fi N=$1 +if [ "$2" = "" ]; then + echo "set instance as argument" +fi +I=$2 PGHOST=`kubectl get services traefik -n system -o jsonpath="{.status.loadBalancer.ingress[0].ip}"` -PGPASSWORD=`kubectl get secrets udi-db-cred -n $N -o jsonpath="{.data.PGPASSWORD}" | base64 --decode` -PGUSER=`kubectl get secrets udi-db-cred -n $N -o jsonpath="{.data.PGUSER}" | base64 --decode` -PGSSLMODE=`kubectl get secrets udi-db-cred -n $N -o jsonpath="{.data.PGSSLMODE}" | base64 --decode` -PGDATABASE=`kubectl get secrets udi-db-cred -n $N -o jsonpath="{.data.PGDATABASE}" | base64 --decode` +PGPASSWORD=`kubectl get secrets $I-udi-db-cred -n $N -o jsonpath="{.data.PGPASSWORD}" | base64 --decode` +PGUSER=`kubectl get secrets $I-udi-db-cred -n $N -o jsonpath="{.data.PGUSER}" | base64 --decode` +PGSSLMODE=`kubectl get secrets $I-udi-db-cred -n $N -o jsonpath="{.data.PGSSLMODE}" | base64 --decode` +PGDATABASE=`kubectl get secrets $I-udi-db-cred -n $N -o jsonpath="{.data.PGDATABASE}" | base64 --decode` export PGUSER PGHOST PGPASSWORD PGSSLMODE PGDATABASE