queries and migrations
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful

This commit is contained in:
Wolfgang Hottgenroth 2024-01-03 20:57:26 +01:00
parent 956d1bdcdb
commit c54b335e5f
Signed by: wn
GPG Key ID: 836E9E1192A6B132
7 changed files with 200 additions and 11 deletions

79
migration/migrate-pv.py Normal file
View File

@ -0,0 +1,79 @@
import psycopg2
from loguru import logger
import os
srcPgHost = os.environ["SRC_PGHOST"]
srcPgUser = os.environ["SRC_PGUSER"]
srcPgPassword = os.environ["SRC_PGPASSWORD"]
srcPgDatabase = os.environ["SRC_PGDATABASE"]
destPgHost = os.environ["DEST_PGHOST"]
destPgUser = os.environ["DEST_PGUSER"]
destPgPassword = os.environ["DEST_PGPASSWORD"]
destPgDatabase = os.environ["DEST_PGDATABASE"]
try:
srcConn = psycopg2.connect(
host=srcPgHost,
dbname=srcPgDatabase,
user=srcPgUser,
password=srcPgPassword,
sslmode='require'
)
srcConn.autocommit = False
destConn = psycopg2.connect(
host=destPgHost,
dbname=destPgDatabase,
user=destPgUser,
password=destPgPassword,
sslmode='require'
)
destConn.autocommit = False
with srcConn.cursor() as srcCur, destConn.cursor() as destCur:
srcCur.execute("select time, deviceid, status, state, importenergyactive, importenergyreactive, exportenergyactive, exportenergyreactive, powerapparent, poweractive, powerreactive, powerdemandpositive, powerdemandreverse, factor, angle, voltage, current, powerdemand from pv_power_measurement order by time")
for srcObj in srcCur:
timestamp = srcObj[0]
deviceName = srcObj[1]
status = srcObj[2]
state = srcObj[3]
importenergyactive = srcObj[4]
importenergyreactive = srcObj[5]
exportenergyactive = srcObj[6]
exportenergyreactive = srcObj[7]
powerapparent = srcObj[8]
poweractive = srcObj[9]
powerreactive = srcObj[10]
powerdemandpositive = srcObj[11]
powerdemandreverse = srcObj[12]
factor = srcObj[13]
angle = srcObj[14]
voltage = srcObj[15]
current = srcObj[16]
powerdemand = srcObj[17]
logger.info(f"{timestamp=}, {deviceName=}")
destTime = timestamp
destApplication = "PV"
destDevice = "Powermeter"
destAttributes = f"{\"ApplicationId\":\"PV\", \"Status\":\"{status}\",\"Hint\": \"Migrated\"}'
destValues = f"{\"Cnt\": {\"unit\": \"\", \"label\": \"\", \"value\": \"-1\", \"variable\": \"Cnt\"}, \"Angle\": {\"unit\": \"degree\", \"label\": \"\", \"value\": \"{angle}\", \"variable\": \"Angle\"}, \"State\": {\"unit\": \"\", \"label\": \"\", \"value\": \"{state}\", \"variable\": \"State\"}, \"Factor\": {\"unit\": \"\", \"label\": \"\", \"value\": \"{factor}\", \"variable\": \"Factor\"}, \"Current\": {\"unit\": \"A\", \"label\": \"\", \"value\": \"{current}\", \"variable\": \"Current\"}, \"Voltage\": {\"unit\": \"V\", \"label\": \"\", \"value\": \"{voltage}\", \"variable\": \"Voltage\"}, \"PowerActive\": {\"unit\": \"W\", \"label\": \"\", \"value\": \"{poweractive}\", \"variable\": \"PowerActive\"}, \"PowerApparent\": {\"unit\": \"VA\", \"label\": \"\", \"value\": \"{powerapparent}\", \"variable\": \"PowerApparent\"}, \"PowerReactive\": {\"unit\": \"VA\", \"label\": \"\", \"value\": \"{powerreactive}\", \"variable\": \"PowerReactive\"}, \"ExportEnergyActive\": {\"unit\": \"Wh\", \"label\": \"\", \"value\": \"{exportenergyactive}\", \"variable\": \"ExportEnergyActive\"}, \"ImportEnergyActive\": {\"unit\": \"Wh\", \"label\": \"\", \"value\": \"{importenergyactive}\", \"variable\": \"ImportEnergyActive\"}, \"PowerDemandReverse\": {\"unit\": \"W\", \"label\": \"\", \"value\": \"{powerdemandreverse}\", \"variable\": \"PowerDemandReverse\"}, \"PowerDemandPositive\": {\"unit\": \"W\", \"label\": \"\", \"value\": \"{powerdemandpositive}\", \"variable\": \"PowerDemandPositive\"}, \"ExportEnergyReactive\": {\"unit\": \"VAh\", \"label\": \"\", \"value\": \"{exportenergyreactive}\", \"variable\": \"ExportEnergyReactive\"}, \"ImportEnergyReactive\": {\"unit\": \"VAh\", \"label\": \"\", \"value\": \"{importenergyreactive}\", \"variable\": \"ImportEnergyReactive\"}}"
logger.info(f"{destTime=}, {destApplication=}, {destDevice=}, {destAttributes=}, {destValues=}")
try:
destCur.execute("insert into measurements (time, application, device, attributes, values) values(%s, %s, %s, %s, %s)",
(destTime, destApplication, destDevice, destAttributes, destValues))
destConn.commit()
except Exception as e:
destConn.rollback()
logger.error(f"Error {e} when inserted time {destTime}")
finally:
if srcConn:
srcConn.close()
if destConn:
destConn.close()

View File

@ -0,0 +1,78 @@
import psycopg2
from loguru import logger
import os
srcPgHost = os.environ["SRC_PGHOST"]
srcPgUser = os.environ["SRC_PGUSER"]
srcPgPassword = os.environ["SRC_PGPASSWORD"]
srcPgDatabase = os.environ["SRC_PGDATABASE"]
destPgHost = os.environ["DEST_PGHOST"]
destPgUser = os.environ["DEST_PGUSER"]
destPgPassword = os.environ["DEST_PGPASSWORD"]
destPgDatabase = os.environ["DEST_PGDATABASE"]
try:
srcConn = psycopg2.connect(
host=srcPgHost,
dbname=srcPgDatabase,
user=srcPgUser,
password=srcPgPassword,
sslmode='require'
)
srcConn.autocommit = False
destConn = psycopg2.connect(
host=destPgHost,
dbname=destPgDatabase,
user=destPgUser,
password=destPgPassword,
sslmode='require'
)
destConn.autocommit = False
with srcConn.cursor() as srcCur, destConn.cursor() as destCur:
srcCur.execute("select time, location, status, temperature, category from room_climate_measurement_t where category = 'heating' and time > '2023-12-19 05:20:00' order by time")
for srcObj in srcCur:
timestamp = srcObj[0]
location = srcObj[1]
status = srcObj[2]
temperature = srcObj[3]
category = srcObj[4]
logger.info(f"{timestamp=}, {location=}, {status=}, {temperature=}, {category=}")
destTime = timestamp
match category:
case 'heating':
destApplication = 'Temperature Heating'
case 'Outdoor':
destApplication = 'Temperature Wago'
case 'Device':
destApplication = 'Temperature Wago'
case 'Indoor':
destApplication = 'Temperature Multisensor' if location != 'Anna-Koeln-2' else 'Temperature Shelly Plus HT'
case 'Special':
destApplication = 'Temperature Multisensor'
destDevice = location
destAttributes = '{"ApplicationId":"temperature-imported", "Status":"' + status + '","Location":"' + location + '","Category":"' + category + '","Hint": "Migrated"}'
destValues = '{"Value": {"unit": "°C", "label": "", "value": "' + str(temperature) + '", "variable": ""}}'
logger.info(f"{destTime=}, {destApplication=}, {destDevice=}, {destAttributes=}, {destValues=}")
try:
destCur.execute("insert into measurements (time, application, device, attributes, values) values(%s, %s, %s, %s, %s)",
(destTime, destApplication, destDevice, destAttributes, destValues))
destConn.commit()
except Exception as e:
destConn.rollback()
logger.error(f"Error {e} when inserted time {destTime}")
finally:
if srcConn:
srcConn.close()
if destConn:
destConn.close()

8
queries/berresheim.sql Normal file
View File

@ -0,0 +1,8 @@
create or replace view level_v as
select time,
cast(values->'CorrectedDistance'->>'value' as float) as level,
cast(values->'Battery'->>'value' as float) as battery,
attributes->>'Status' as status,
device
from measurements
where application = 'de-hottis-level-monitoring';

View File

@ -45,6 +45,13 @@ create or replace view temperature_v as
from measurements from measurements
where application in ('Temperature Multisensor', 'Temperature Shelly Plus HT'); where application in ('Temperature Multisensor', 'Temperature Shelly Plus HT');
create or replace view temperature2_v as
select time,
cast(values->'Value'->>'value' as float) as temperature,
device
from measurements
where application = 'Temperature Wago';
create or replace view humidity_v as create or replace view humidity_v as
select time, select time,
cast(values->'Value'->>'value' as float) as humidity, cast(values->'Value'->>'value' as float) as humidity,
@ -52,3 +59,11 @@ create or replace view humidity_v as
from measurements from measurements
where application in ('Humidity Multisensor'); where application in ('Humidity Multisensor');
create or replace view soil_v as
select time,
cast(values->'Water'->>'value' as float) as water,
cast(values->'Conductance'->>'value' as float) as conductance,
cast(values->'Temperature'->>'value' as float) as temperature,
device
from measurements
where application = 'de-hottis-app01' and attributes->>'DeviceType' = 'dragino-lse01';

View File

@ -0,0 +1,11 @@
select
extract('day' from time)::varchar || '.' || extract('month' from time)::varchar || '.' || extract('year' from time)::varchar as day,
avg(temperature)::numeric(10,0) as temperature
from room_climate_measurement_t
where
category = 'Outdoor' and
location = 'Outdoor' and
extract('hour' from time) = 12 and
time::date = now()::date
group by day

View File

@ -3,13 +3,7 @@ create or replace view power_v as
cast(values->'ActivePowerL1'->>'value' as float) as power_l1, cast(values->'ActivePowerL1'->>'value' as float) as power_l1,
cast(values->'ActivePowerL2'->>'value' as float) as power_l2, cast(values->'ActivePowerL2'->>'value' as float) as power_l2,
cast(values->'ActivePowerL3'->>'value' as float) as power_l3, cast(values->'ActivePowerL3'->>'value' as float) as power_l3,
device cast(values->'ActivePowerL123'->>'value' as float) as power_total,
from measurements
where application = 'com-passavant-geiger-poc' and
attributes->>'FPort' = '1';
create or replace view power_factor_v as
select time,
cast(values->'PowerfactorL1'->>'value' as float) as factor_l1, cast(values->'PowerfactorL1'->>'value' as float) as factor_l1,
cast(values->'PowerfactorL2'->>'value' as float) as factor_l2, cast(values->'PowerfactorL2'->>'value' as float) as factor_l2,
cast(values->'PowerfactorL3'->>'value' as float) as factor_l3, cast(values->'PowerfactorL3'->>'value' as float) as factor_l3,

View File

@ -2,9 +2,13 @@ if [ "$1" = "" ]; then
echo "set namespace as argument" echo "set namespace as argument"
fi fi
N=$1 N=$1
if [ "$2" = "" ]; then
echo "set instance as argument"
fi
I=$2
PGHOST=`kubectl get services traefik -n system -o jsonpath="{.status.loadBalancer.ingress[0].ip}"` PGHOST=`kubectl get services traefik -n system -o jsonpath="{.status.loadBalancer.ingress[0].ip}"`
PGPASSWORD=`kubectl get secrets udi-db-cred -n $N -o jsonpath="{.data.PGPASSWORD}" | base64 --decode` PGPASSWORD=`kubectl get secrets $I-udi-db-cred -n $N -o jsonpath="{.data.PGPASSWORD}" | base64 --decode`
PGUSER=`kubectl get secrets udi-db-cred -n $N -o jsonpath="{.data.PGUSER}" | base64 --decode` PGUSER=`kubectl get secrets $I-udi-db-cred -n $N -o jsonpath="{.data.PGUSER}" | base64 --decode`
PGSSLMODE=`kubectl get secrets udi-db-cred -n $N -o jsonpath="{.data.PGSSLMODE}" | base64 --decode` PGSSLMODE=`kubectl get secrets $I-udi-db-cred -n $N -o jsonpath="{.data.PGSSLMODE}" | base64 --decode`
PGDATABASE=`kubectl get secrets udi-db-cred -n $N -o jsonpath="{.data.PGDATABASE}" | base64 --decode` PGDATABASE=`kubectl get secrets $I-udi-db-cred -n $N -o jsonpath="{.data.PGDATABASE}" | base64 --decode`
export PGUSER PGHOST PGPASSWORD PGSSLMODE PGDATABASE export PGUSER PGHOST PGPASSWORD PGSSLMODE PGDATABASE