From 0f0da63a8f73626821158022dea1bf1720c61636 Mon Sep 17 00:00:00 2001 From: Wolfgang Ludger Hottgenroth Date: Tue, 2 Dec 2025 14:49:51 +0100 Subject: [PATCH] initial for multiple devices, introduce real configuration --- .gitlab-ci.yml | 5 -- .woodpecker.yml | 4 +- LICENSE | 21 ----- config/config.yaml | 134 ++++++++++++++++++++++++++++++ requirements.txt | 12 +++ schema/all-in-one-yield.sql | 67 --------------- schema/create.sql | 88 -------------------- schema/queries01.sql | 160 ------------------------------------ schema/queries02.sql | 20 ----- schema/yield-by-month.sql | 0 src/pv_controller/config.py | 101 ++++++++++++++++------- 11 files changed, 220 insertions(+), 392 deletions(-) delete mode 100644 .gitlab-ci.yml delete mode 100644 LICENSE create mode 100644 config/config.yaml create mode 100644 requirements.txt delete mode 100644 schema/all-in-one-yield.sql delete mode 100644 schema/create.sql delete mode 100644 schema/queries01.sql delete mode 100644 schema/queries02.sql delete mode 100644 schema/yield-by-month.sql diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 0c79616..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,5 +0,0 @@ -include: - - project: dockerized/commons - ref: master - file: gitlab-ci-template.yml - diff --git a/.woodpecker.yml b/.woodpecker.yml index 45910ee..2e80e27 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -5,14 +5,14 @@ steps: repo: gitea.hottis.de/wn/pv-controller registry: from_secret: container_registry - tags: latest,${CI_COMMIT_SHA},${CI_COMMIT_TAG} + tags: latest,${CI_COMMIT_TAG} username: from_secret: container_registry_username password: from_secret: container_registry_password dockerfile: Dockerfile when: - - event: [push, tag] + - event: tag deploy: image: portainer/kubectl-shell:latest diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 518ef33..0000000 --- a/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2022 Wolfgang Hottgenroth - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/config/config.yaml b/config/config.yaml new file mode 100644 index 0000000..d643cf6 --- /dev/null +++ b/config/config.yaml @@ -0,0 +1,134 @@ +mqtt: + broker: ${MQTT__BROKER} + port: ${MQTT__PORT} + +modbus: + gateway: ${MODBUS__GATEWAY} + + +# REGISTERS = [ +# { "slave":2, "addr":0x0048, "type":"input", "attr": "importEnergyActive", "name":"Import active energy", "unit":"kWh", "adaptor": floatAdaptor }, +# { "slave":2, "addr":0x004c, "type":"input", "attr": "importEnergyReactive", "name":"Import reactive energy", "unit":"kVAh", "adaptor": floatAdaptor }, +# { "slave":2, "addr":0x004a, "type":"input", "attr": "exportEnergyActive", "name":"Export active energy", "unit":"kWh", "adaptor": floatAdaptor }, +# { "slave":2, "addr":0x004e, "type":"input", "attr": "exportEnergyReactive", "name":"Export reactive energy", "unit":"kVAh", "adaptor": floatAdaptor }, +# { "slave":2, "addr":0x0012, "type":"input", "attr": "powerApparent", "name":"Apparent Power", "unit":"W", "adaptor": floatAdaptor }, +# { "slave":2, "addr":0x000c, "type":"input", "attr": "powerActive", "name":"Active Power", "unit":"W", "adaptor": floatAdaptor }, +# { "slave":2, "addr":0x0018, "type":"input", "attr": "powerReactive", "name":"Reactive Power", "unit":"W", "adaptor": floatAdaptor }, +# { "slave":2, "addr":0x0058, "type":"input", "attr": "powerDemandPositive", "name":"PositivePowerDemand", "unit":"W", "adaptor": floatAdaptor }, +# { "slave":2, "addr":0x005c, "type":"input", "attr": "powerDemandReverse", "name":"ReversePowerDemand", "unit":"W", "adaptor": floatAdaptor }, +# { "slave":2, "addr":0x001e, "type":"input", "attr": "factor", "name":"Factor", "unit":"-", "adaptor": floatAdaptor }, +# { "slave":2, "addr":0x0024, "type":"input", "attr": "angle", "name":"Angle", "unit":"degree", "adaptor": floatAdaptor }, +# { "slave":2, "addr":0x0000, "type":"input", "attr": "voltage", "name":"Voltage", "unit":"V", "adaptor": floatAdaptor }, +# { "slave":2, "addr":0x0006, "type":"input", "attr": "current", "name":"Current", "unit":"A", "adaptor": floatAdaptor }, +# { "slave":1, "addr":0x0001, "type":"holding", "attr": "state", "name":"State", "unit":"-", "adaptor": onOffAdaptor }, +# ] + +output: + - name: pv_meter + publish_topic: IoT/PV/Values + publish_period: 15 + slave_id: 2 + registers: + - address: 0x0048 + attribute: importEnergyActive + name: Import active energy + unit: kWh + register_type: input + data_type: float + adaptor: floatAdaptor + - address: 0x004c + attribute: importEnergyReactive + name: Import reactive energy + unit: kVAh + register_type: input + data_type: float + adaptor: floatAdaptor + - address: 0x004a + attribute: exportEnergyActive + name: Export active energy + unit: kWh + register_type: input + data_type: float + adaptor: floatAdaptor + - address: 0x004e + attribute: exportEnergyReactive + name: Export reactive energy + unit: kVAh + register_type: input + data_type: float + adaptor: floatAdaptor + - address: 0x0012 + attribute: powerApparent + name: Apparent Power + unit: W + register_type: input + data_type: float + adaptor: floatAdaptor + - address: 0x000c + attribute: powerActive + name: Active Power + unit: W + register_type: input + data_type: float + adaptor: floatAdaptor + - address: 0x0018 + attribute: powerReactive + name: Reactive Power + unit: W + register_type: input + data_type: float + adaptor: floatAdaptor + - address: 0x0058 + attribute: powerDemandPositive + name: PositivePowerDemand + unit: W + register_type: input + data_type: float + adaptor: floatAdaptor + - address: 0x005c + attribute: powerDemandReverse + name: ReversePowerDemand + unit: W + register_type: input + data_type: float + adaptor: floatAdaptor + - address: 0x001e + attribute: factor + name: Factor + unit: "-" + register_type: input + data_type: float + adaptor: floatAdaptor + - address: 0x0024 + attribute: angle + name: Angle + unit: degree + register_type: input + data_type: float + adaptor: floatAdaptor + - address: 0x0000 + attribute: voltage + name: Voltage + unit: V + register_type: input + data_type: float + adaptor: floatAdaptor + - address: 0x0006 + attribute: current + name: Current + unit: A + register_type: input + data_type: float + adaptor: floatAdaptor + - name: pv_control + publish_topic: IoT/PV/Control + publish_period: 15 + slave_id: 1 + registers: + - address: 0x0001 + attribute: state + name: State + unit: "-" + register_type: holding + data_type: int + adaptor: onOffAdaptor diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a9360e2 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +# Configuration and validation +pydantic>=2.0.0 +pyyaml>=6.0 + +# Logging +loguru>=0.7.0 + +# MQTT client +paho-mqtt>=1.6.0 + +# Modbus communication +pymodbus>=3.0.0 diff --git a/schema/all-in-one-yield.sql b/schema/all-in-one-yield.sql deleted file mode 100644 index 1d4f7f5..0000000 --- a/schema/all-in-one-yield.sql +++ /dev/null @@ -1,67 +0,0 @@ -with - first_day_in_year as ( - select - date_trunc('day', min(time)) as day - from pv_power_measurement_t - where - time between date_trunc('year', time) and now() - ), - first_value_in_year as ( - select - time_bucket('1 day', time) as interval, - first(exportenergyactive, time) as energy - from pv_power_measurement_t - where - time between (select day from first_day_in_year) and (select day from first_day_in_year) + interval '1 day' and - status = 'Ok' - group by interval - ), - first_day_in_month as ( - select - date_trunc('day', min(time)) as day - from pv_power_measurement_t - where - time between date_trunc('month', now()) and now() - ), - first_value_in_month as ( - select - time_bucket('1 day', time) as interval, - first(exportenergyactive, time) as energy - from pv_power_measurement_t - where - time between (select day from first_day_in_month) and (select day from first_day_in_month) + interval '1 day' and - status = 'Ok' - group by interval - ), - first_value_in_day as ( - select - time_bucket('1 day', time) as interval, - first(exportenergyactive, time) as energy - from pv_power_measurement_t - where time >= date_trunc('day', now()) - group by interval - ), - last_value as ( - select - time_bucket('1 day', time) as interval, - last(exportenergyactive, time) as energy - from pv_power_measurement_t - where - time between date_trunc('day', now()) and date_trunc('day', now()) + interval '1 day' and - status = 'Ok' - group by interval - ) - select - extract(year from (select day from first_day_in_year))::text as period_value, - 'Year' as period_name, - round(((select energy from last_value) - (select energy from first_value_in_year))::numeric, 2) as yield - union - select - to_char((select day from first_day_in_month), 'Month') as period_value, - 'Month' as period_name, - round(((select energy from last_value) - (select energy from first_value_in_month))::numeric, 2) as yield - union - select - now()::date::text as period_value, - 'Day' as period_name, - round(((select energy from last_value) - (select energy from first_value_in_day))::numeric, 2) as yield; diff --git a/schema/create.sql b/schema/create.sql deleted file mode 100644 index cca7b07..0000000 --- a/schema/create.sql +++ /dev/null @@ -1,88 +0,0 @@ -create table pv_power_measurement_t ( - time timestamp without time zone not null, - deviceid text, - status text, - state integer, - importEnergyActive double precision, - importEnergyReactive double precision, - exportEnergyActive double precision, - exportEnergyReactive double precision, - powerApparent double precision, - powerActive double precision, - powerReactive double precision, - powerDemandPositive double precision, - powerDemandReverse double precision, - powerDemand double precision, - factor double precision, - angle double precision, - voltage double precision, - current double precision -); - -select create_hypertable('pv_power_measurement_t', 'time'); - -grant insert on pv_power_measurement_t to nodered; -grant select on pv_power_measurement_t to grafana; - -create view pv_stats_v as - select time, importEnergyActive, importEnergyReactive, exportEnergyActive, exportEnergyReactive, - powerApparent, powerActive, powerReactive, powerDemandPositive, powerDemandReverse, powerDemand, - factor, angle, voltage, current - from pv_power_measurement_t - order by time; - - -create table pv_stats_t ( - id serial not null primary key, - "date" date not null, - dateType varchar(5) not null, - first numeric(10,2) not null default 0, - total numeric(10,2) not null default 0 -); -alter table pv_stats_t add constraint ddT_uk unique ("date", dateType); - -grant insert, select, update on pv_stats_t to nodered; -grant select, update on pv_stats_t_id_seq to nodered; - -create or replace function pv_stats_func () - returns trigger - language plpgsql -as $$ -declare - v_stat_id pv_stats_t.id%TYPE; - v_dateTypes varchar[] := array['day', 'month', 'year']; - v_dateType varchar; -begin - foreach v_dateType in array v_dateTypes - loop - select id - from pv_stats_t - into v_stat_id - where "date" = date_trunc(v_dateType, NEW.time::date) and - dateType = v_dateType; - if not found then - insert into pv_stats_t ("date", dateType, first) - values (date_trunc(v_dateType, NEW.time::date), v_dateType, NEW.exportEnergyActive); - else - update pv_stats_t - set total = NEW.exportEnergyActive - first - where id = v_stat_id; - end if; - - end loop; - - return NEW; -end; -$$ - -create trigger pv_stats_trig - after insert on pv_power_measurement_t - for each row - execute function pv_stats_func(); - - - -insert into pv_stats_t("date", dateType, first, total) values (date_trunc('month', now()), 'month', 0.01, 0) - on conflict on constraint ddT_uk do update set total = 3.26 - excluded.first; - -; diff --git a/schema/queries01.sql b/schema/queries01.sql deleted file mode 100644 index cb8cf92..0000000 --- a/schema/queries01.sql +++ /dev/null @@ -1,160 +0,0 @@ --- current year's gain -with - first_day_in_year as ( - select - date_trunc('day', min(time)) as day - from pv_power_measurement_t - where - time between date_trunc('year', time) and now() - ), - first_value as ( - select - time_bucket('1 day', time) as interval, - first(exportenergyactive, time) as energy - from pv_power_measurement_t - where - time between (select day from first_day_in_year) and (select day from first_day_in_year) + interval '1 day' and - status = 'Ok' - group by interval - ), - last_value as ( - select - time_bucket('1 day', time) as interval, - last(exportenergyactive, time) as energy - from pv_power_measurement_t - where - time between date_trunc('day', now()) and date_trunc('day', now()) + interval '1 day' and - status = 'Ok' - group by interval - ) - select - extract(year from (select day from first_day_in_year))::text as period_value, - 'Year' as period_name, - (select energy from last_value) - (select energy from first_value) as yield; - - - --- current month's gain -with - first_day_in_month as ( - select - date_trunc('day', min(time)) as day - from pv_power_measurement_t - where - time between date_trunc('month', now()) and now() - ), - first_value as ( - select - time_bucket('1 day', time) as interval, - first(exportenergyactive, time) as energy - from pv_power_measurement_t - where - time between (select day from first_day_in_month) and (select day from first_day_in_month) + interval '1 day' and - status = 'Ok' - group by interval - ), - last_value as ( - select - time_bucket('1 day', time) as interval, - last(exportenergyactive, time) as energy - from pv_power_measurement_t - where - time between date_trunc('day', now()) and date_trunc('day', now()) + interval '1 day' and - status = 'Ok' - group by interval - ) - select - (select day from first_day_in_month) as v1, - (select energy from first_value) as v2, - (select energy from last_value) as v3, - to_char((select day from first_day_in_month), 'Month') as period_value, - 'Month' as period_name, - (select energy from last_value) - (select energy from first_value) as yield; - - - --- current day's gain -with - values as ( - select - time_bucket('1 day', time) as interval, - first(exportenergyactive, time) as first_value, - last(exportenergyactive, time) as last_value - from pv_power_measurement_t - where time >= date_trunc('day', now()) - group by interval - ) - select - (select interval from values)::date::text as period_value, - 'Day' as period_name, - (select last_value from values) - (select first_value from values) as yield; - - --- all in one -with - first_day_in_year as ( - select - date_trunc('day', min(time)) as day - from pv_power_measurement_t - where - time between date_trunc('year', time) and now() - ), - first_value_in_year as ( - select - time_bucket('1 day', time) as interval, - first(exportenergyactive, time) as energy - from pv_power_measurement_t - where - time between (select day from first_day_in_year) and (select day from first_day_in_year) + interval '1 day' and - status = 'Ok' - group by interval - ), - first_day_in_month as ( - select - date_trunc('day', min(time)) as day - from pv_power_measurement_t - where - time between date_trunc('month', time) and now() - ), - first_value_in_month as ( - select - time_bucket('1 day', time) as interval, - first(exportenergyactive, time) as energy - from pv_power_measurement_t - where - time between (select day from first_day_in_month) and (select day from first_day_in_month) + interval '1 day' and - status = 'Ok' - group by interval - ), - first_value_in_day as ( - select - time_bucket('1 day', time) as interval, - first(exportenergyactive, time) as energy - from pv_power_measurement_t - where time >= date_trunc('day', now()) - group by interval - ), - last_value as ( - select - time_bucket('1 day', time) as interval, - last(exportenergyactive, time) as energy - from pv_power_measurement_t - where - time between date_trunc('day', now()) and date_trunc('day', now()) + interval '1 day' and - status = 'Ok' - group by interval - ) - select - extract(year from (select day from first_day_in_year))::text as period_value, - 'Year' as period_name, - round(((select energy from last_value) - (select energy from first_value_in_year))::numeric, 2) as yield - union - select - to_char((select day from first_day_in_month), 'Month') as period_value, - 'Month' as period_name, - round(((select energy from last_value) - (select energy from first_value_in_month))::numeric, 2) as yield - union - select - now()::date::text as period_value, - 'Day' as period_name, - round(((select energy from last_value) - (select energy from first_value_in_day))::numeric, 2) as yield; diff --git a/schema/queries02.sql b/schema/queries02.sql deleted file mode 100644 index 0424e61..0000000 --- a/schema/queries02.sql +++ /dev/null @@ -1,20 +0,0 @@ -select time_bucket('1 day', time) as interval, - round((last(exportenergyactive, time) - first(exportenergyactive, time))::numeric, 2) as energy - from pv_power_measurement_t - where time between date_trunc('month', now()) and date_trunc('month', now()) + interval '1 month' - group by interval - order by interval; - - - --- daily stats of current month - - - - -select time_bucket('1 day', time) as interval, - round((last(exportenergyactive, time) - first(exportenergyactive, time))::numeric, 2) as energy - from pv_power_measurement_t - where time between date_trunc('month', now()) and date_trunc('month', now()) + interval '1 month' - group by interval - order by interval; \ No newline at end of file diff --git a/schema/yield-by-month.sql b/schema/yield-by-month.sql deleted file mode 100644 index e69de29..0000000 diff --git a/src/pv_controller/config.py b/src/pv_controller/config.py index 213da3f..5dc4f9c 100644 --- a/src/pv_controller/config.py +++ b/src/pv_controller/config.py @@ -1,33 +1,76 @@ import os +import re +from pathlib import Path +from typing import List, Optional +from pydantic import BaseModel, Field, field_validator +import yaml from loguru import logger -class Config: - OPTIONS = { - 'mqtt': [ 'login', - 'password', - 'ca', - 'cert', - 'key', - 'broker', - 'port', - 'meterPublishTopic', - 'meterPublishPeriod', - 'relaisSubscribeTopic' ], - 'modbus': [ 'gateway' ] - } + +class RegisterConfig(BaseModel): + """Modbus Register Configuration""" + address: int + attribute: str + name: str + unit: str + register_type: str + data_type: str + adaptor: str + + +class OutputConfig(BaseModel): + """Output Configuration for Modbus Devices""" + name: str + publish_topic: str + publish_period: int + slave_id: int + registers: List[RegisterConfig] + + +class MqttConfig(BaseModel): + """MQTT Configuration""" + broker: str + port: int + + +class ModbusConfig(BaseModel): + """Modbus Configuration""" + gateway: str + + +class Config(BaseModel): + """Main Configuration""" + mqtt: MqttConfig + modbus: ModbusConfig + output: List[OutputConfig] + + @classmethod + def load_from_file(cls, config_path: Optional[str] = None) -> 'Config': + """ + Load configuration from YAML file with environment variable substitution. + + Args: + config_path: Path to config file. If None, uses CFG_FILE environment variable. + + Returns: + Config instance + """ + if config_path is None: + config_path = os.getenv('CFG_FILE') + if config_path is None: + raise ValueError("Config path not provided and CFG_FILE environment variable not set") + + config_file = Path(config_path) + if not config_file.exists(): + raise FileNotFoundError(f"Configuration file not found: {config_path}") + + # Read YAML file + with open(config_file, 'r', encoding='utf-8') as f: + yaml_content = f.read() + + # Parse YAML + config_dict = yaml.safe_load(yaml_content) + + logger.info(f"Configuration loaded from: {config_path}") + return cls(**config_dict) - def __init__(self): - self.values = {} - for section, keys in Config.OPTIONS.items(): - self.values[section] = {} - for key in keys: - varname = f"{section}__{key}".upper() - try: - self.values[section][key] = os.environ[varname] - logger.info(f"Config: {section} {key} -> {self.values[section][key]}") - except KeyError: - pass - - def __getitem__(self, section): - return self.values[section] -