rollback to old database

.woodpecker.yml

@@ -4,53 +4,18 @@ steps:
     settings:
       repo: ${FORGE_NAME}/${CI_REPO}
       registry:
-        from_secret: container_registry
-      tags: latest,${CI_COMMIT_SHA},${CI_COMMIT_TAG}
+        from_secret: local_registry
+      tags: latest,${CI_COMMIT_TAG}
       username:
-        from_secret: container_registry_username
+        from_secret: local_username
       password:
-        from_secret: container_registry_password
+        from_secret: local_password
       dockerfile: Dockerfile
     when:
-      - event: [push, tag]
-        
-  scan:
-    image: quay.io/wollud1969/woodpecker-helper:0.5.1
-    environment:
-      TRIVY_TOKEN:
-        from_secret: trivy_token
-      TRIVY_URL:
-        from_secret: trivy_url
-      DTRACK_API_KEY:
-        from_secret: dtrack_api_key
-      DTRACK_API_URL:
-        from_secret: dtrack_api_url
-    commands:
-      - HOME=/home/`id -nu`
-      - TAG="${CI_COMMIT_TAG:-$CI_COMMIT_SHA}"
-      - |
-        trivy image \
-              --server $TRIVY_URL \
-              --token $TRIVY_TOKEN \
-              --format cyclonedx \
-              --scanners license \
-              --output /tmp/sbom.xml \
-              $FORGE_NAME/$CI_REPO:$TAG
-      - cat /tmp/sbom.xml
-      - |
-        curl -X "POST" \
-             -H "Content-Type: multipart/form-data" \
-             -H "X-Api-Key: $DTRACK_API_KEY" \
-             -F "autoCreate=true" \
-             -F "projectName=$CI_REPO" \
-             -F "projectVersion=$TAG" \
-             -F "bom=@/tmp/sbom.xml"\
-             "$DTRACK_API_URL/api/v1/bom"        
-    when:
-      - event: [push, tag]
+      - event: tag
         
   deploy:
-    image: quay.io/wollud1969/k8s-admin-helper:0.2.1
+    image: quay.io/wollud1969/k8s-admin-helper:0.4.1
     environment:
       KUBE_CONFIG_CONTENT:
         from_secret: kube_config

debug-build-run.sh

@@ -7,22 +7,7 @@ IMAGE_NAME=numberimage
 docker build --progress=plain -t $IMAGE_NAME .
 
 
-SECRETS=`mktemp`
-gpg --decrypt --passphrase $GPG_PASSPHRASE --yes --batch --output $SECRETS ./deployment/secrets.asc
-. $SECRETS
-rm $SECRETS
-
-DB_NAMESPACE=database1
-DB_DEPLOYNAME=database
-
-REDIS_NAMESPACE=redis
-REDIS_SERVICE_NAME=redis
-
-PGHOST=`kubectl get services $DB_DEPLOYNAME -n $DB_NAMESPACE -o jsonpath="{.status.loadBalancer.ingress[0].ip}"`
-REDISHOST=`kubectl get services $REDIS_SERVICE_NAME -n $REDIS_NAMESPACE -o jsonpath="{.status.loadBalancer.ingress[0].ip}"`
-
-REDIS_URL=redis://$REDISHOST:6379/4
-
+. load-debug-env
 
 docker run \
   -it \

deployment/encrypt-secrets.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+gpg --pinentry-mode=loopback --symmetric --cipher-algo AES256 --armor --output secrets.asc secrets.txt
+

deployment/secrets.asc

@@ -1,15 +1,15 @@
 -----BEGIN PGP MESSAGE-----
 
-jA0ECQMC4tvOFCp2PT7/0sE+AcZmiGwroHYdWW/vJA1sCoMrM2oW8xUc1KndDGto
-dFev2KcoZ4FRL9liCrJ7on773bZFCTDu2xiBNMeKF1p8drub1Ej5l61Oq15mLTYf
-XjXknFiWWq4PTzhhy53zvDyIV5yIcGfjotpGC83/qH5CBWIcCw9PZHI5+uHRVRKL
-OpdTpekJ8ljPAGQ4F3vbOeBbG1PMoclx8r5SpKxLwQco2iaXd71dXHtPkEnLujd4
-ZRCthVNVfxrIIRcTJfgxewz2oJWYF9008nmtxpKzqfBtNpFIdBtnTTVvv2lfcVuS
-S4eav9ljPPd/exZaT7fOGD/kuCdiiu6e0yGLzo2ykf0uBy7hG7ZJg8TH5e+LMBm9
-Q1OFD+5LWeEsOEdSTlT5UbV5EaVEiawKWQn7rMZOyvBNiPwLaOlquHirWoll5eIm
-OHgBVN0FiY/righpIoei/KP705FIg+hrpMUvc29PahiL8dgsxJycnKIo4t+2/nac
-H144hP/rqBeaobG05TkZIr+Cpt4rpwwfNWOHgmNFHVpxdmPfoeAmpT4nz760hTEN
-2ol4Qh8xL7n4GFiCpNg8tNZiZkPPa/aRUAFxgdq5beossvKukxCexQmlCFvxITSG
-x9RwssFMnT+wqTuzBN8neBeEF1d9AtAFQKPtg3wkMUyJAlYkxiGS/2NJuYDVpmEQ
-=Wr4f
+jA0ECQMIiv7iz3c3MUj+0sE1AWdfA7cItvT1AKu3kypomk9QYoEttRzH0AePc5No
+J2KfMlJ7ZKvWMjLIKLZ+qr4syI62FMozWM+iWC++WC6Bjy2z4etrDkRYluYioiuk
+pUSh1UWXn1F9mk33s9bXO4C/TJXQivtA7K0o1vD8YCWkNApc+wzxOnp373Bba9kb
+l1VVMRhUdobfTOmuubtVb/ioeC5Djxv8K59UWM3zs64Mpm4ctGIMiXIC2SQjMcga
+j4EjGq16qBhzOT5kwhEvg90Ksjej2rzk/cqfdc72zK1p5jSWAyeBUZva3qoBkCc/
+ZgZ+PuXZDW1PO/0yTaIob0SfFrHFmnAGA074VEPlg8EJlHMZjd0eVfISwsIipBg7
+rpPyyEWkMSGhnObOX/C9lGbi3/3JNbH/Q2xfkTmjRtnTjpJ3CRsW8Dek59rTfVsu
+dr7Z0gzLo7nDQRTraIfEpgS7xwjOWLV0iA5Fwjl9Vlg1WDao7RA0+uSPZwIxdnSS
+ma/Lj8XNaPYzmyeDFMcCcBaIXj9PzdONNxfKhqn57aA/94M7aUdgYSHfhgqdGBRW
+gE55Eyz+S0RgNiRGo2VIJudoUM3IWr0XM/DR943JEYs796TEUncd1oRHoJNRW5Se
+UtJurdyXfPqXPvyrj+4sqo/k8RhRdJW3EasuEfu1SHsK7EltDecZ
+=KuBv
 -----END PGP MESSAGE-----

load-debug-env

@@ -0,0 +1,15 @@
+SECRETS=`mktemp`
+gpg --decrypt --passphrase $GPG_PASSPHRASE --yes --batch --output $SECRETS ./deployment/secrets.asc
+. $SECRETS
+rm $SECRETS
+
+DB_NAMESPACE=database1
+DB_DEPLOYNAME=database
+
+REDIS_NAMESPACE=redis
+REDIS_SERVICE_NAME=redis
+
+PGHOST=`kubectl get services $DB_DEPLOYNAME -n $DB_NAMESPACE -o jsonpath="{.status.loadBalancer.ingress[0].ip}"`
+REDISHOST=`kubectl get services $REDIS_SERVICE_NAME -n $REDIS_NAMESPACE -o jsonpath="{.status.loadBalancer.ingress[0].ip}"`
+
+REDIS_URL=redis://$REDISHOST:6379/4

src/ntp_routes.py

@@ -1,157 +1,69 @@
-from flask import Flask, session, g, render_template_string
+from flask import Flask, session, g, render_template_string, Response
 from loguru import logger
 import json
-import plotly.express as px
-import plotly.graph_objects as po
 import matplotlib.pyplot as plt
+import matplotlib.dates as mdates
+from matplotlib.ticker import ScalarFormatter
 import pandas as pd
 import psycopg
 import sqlalchemy
 import time
+import io
 
 from app import app
 from app import oidc
 
 
 
-@app.route('/ntpserver')
-def ntpserver():
-    try:
-        dbh = psycopg.connect()
-        engine = sqlalchemy.create_engine("postgresql+psycopg://", creator=lambda: dbh)
 
-        query = """
-          select   time_bucket('5 minutes', time) as bucket,
-            device,
-            avg(cast(values->'rootdisp'->>'value'  as float)) as rootdisp,
-            max(cast(values->'stratum'->>'value'  as int)) as stratum
-          from measurements
-          where time >= date_trunc('day', now()) AND time < date_trunc('day', now()) + '1 day'::interval and 
-            application = 'TSM' and attributes->>'Label' = 'david'
-          group by bucket, device
-          order by bucket, device
-        """      
-
-        df = pd.read_sql(query, con=engine)
-
-        fig = po.Figure()
-        fig.add_trace(po.Scatter(x=df['bucket'], y=df['rootdisp'], mode='lines', name='Root Dispersion', yaxis='y1', line=dict(color='red')))
-        fig.add_trace(po.Scatter(x=df['bucket'], y=df['stratum'], mode='lines', name='Stratum', yaxis='y2', line=dict(color='blue')))
-
-        fig.update_layout(
-            title='NTP Server Numbers',
-            
-            # Linke Y-Achse
-            yaxis=dict(
-                title='Root Dispersion',
-                ticksuffix=' ms'
-            ),
-            
-            # Rechte Y-Achse
-            yaxis2=dict(
-                title='Stratum',
-                overlaying='y',  # Legt die zweite Y-Achse über die erste
-                side='right',  # Setzt sie auf die rechte Seite
-                tickmode='linear',  # Stellt sicher, dass die Ticks in festen Intervallen sind
-                dtick=1,  # Zeigt nur ganzzahlige Ticks
-            ),
-            
-            legend=dict(x=0.05, y=1)  # Position der Legende
-        )
-
-        graph_html_1 = fig.to_html(full_html=False, default_height='30%')
-
-        query = """
-            select   time_bucket('5 minutes', time) as bucket,
-              device,
-              avg(cast(values->'time-req-pkts'->>'value'  as float)) as packets
-            from measurements
-            where time >= date_trunc('day', now()) AND time < date_trunc('day', now()) + '1 day'::interval and 
-              application = 'SNMP' and attributes->>'Label' = 'david'
-            group by bucket, device
-            order by bucket, device
-        """
-        df = pd.read_sql(query, con=engine)
-        fig_2 = px.line(df, x='bucket', y='packets')
-        fig_2.update_layout(
-            xaxis_title="",
-            yaxis_title="",
-            yaxis_ticksuffix="p/s",
-            title=f"Time Requests"
-        )
-        graph_html_2 = fig_2.to_html(full_html=False, default_height='30%')
-
-        query = """
-            select   time_bucket('5 minutes', time) as bucket,
-              device,
-              avg(cast(values->'load1'->>'value'  as float)) as loadaverage1min
-            from measurements
-            where time >= date_trunc('day', now()) AND time < date_trunc('day', now()) + '1 day'::interval and 
-              application = 'SNMP' and attributes->>'Label' = 'david'
-            group by bucket, device
-            order by bucket, device
-        """
-        df = pd.read_sql(query, con=engine)
-        fig_3 = px.line(df, x='bucket', y='loadaverage1min')
-        fig_3.update_layout(
-            xaxis_title="",
-            yaxis_title="",
-            title=f"CPU Load"
-        )
-        graph_html_3 = fig_3.to_html(full_html=False, default_height='30%')
-
-        return render_template_string(f"""
-            <html>
-                <head>
-                    <title>NTP Server Numbers</title>
-                </head>
-                <body>
-                    {graph_html_1}
-                    {graph_html_2}
-                    {graph_html_3}
-                </body>
-            </html>
-        """)
-    except Exception as e:
-        raise Exception(f"Error when querying NTP server values: {e}")
-    finally:
-        if dbh is not None:
-            dbh.close()
-
-
-def get_dataframe():
-
-@app.route('/plot.png')
-def plot_png():
+@app.route('/ntp/stratum-rootdisp.png')
+def stratum_rootdisp_png():
     dbh = psycopg.connect()
     engine = sqlalchemy.create_engine("postgresql+psycopg://", creator=lambda: dbh)
     query = """
       select   time_bucket('5 minutes', time) as bucket,
-        device,
+        attributes->>'Label' as device,
         avg(cast(values->'rootdisp'->>'value'  as float)) as rootdisp,
         max(cast(values->'stratum'->>'value'  as int)) as stratum
       from measurements
       where time >= date_trunc('day', now()) AND time < date_trunc('day', now()) + '1 day'::interval and
-        application = 'TSM' and attributes->>'Label' = 'david'
-      group by bucket, device
-      order by bucket, device
+        application = 'SNMP' and attributes->>'Label' IN ('harrison', 'david')
+      group by bucket, attributes->>'Label'
+      order by bucket, attributes->>'Label'
     """
     df = pd.read_sql(query, con=engine)
     
-    fig, ax1 = plt.subplots()
+    df['rootdisp'] = df['rootdisp'] / 1e6
 
-    ax1.plot(df['bucket'], df['rootdisp'], 'r-', label='Root Dispersion')
+    
+    # Extract date for title
+    plot_date = df['bucket'].dt.date.iloc[0] if not df.empty else "Unknown Date"
+    
+    # Create figure with two side-by-side subplots
+    fig, axes = plt.subplots(1, 2, figsize=(15, 5), sharex=True)
+    
+    for i, device in enumerate(['harrison', 'david']):
+        ax1 = axes[i]
+        ax2 = ax1.twinx()
+        
+        device_df = df[df['device'] == device]
+        
+        ax1.plot(device_df['bucket'], device_df['rootdisp'], 'r-', label='Root Dispersion')
         ax1.set_xlabel('Time')
         ax1.set_ylabel('Root Dispersion (ms)', color='r')
         ax1.tick_params(axis='y', labelcolor='r')
         
-    ax2 = ax1.twinx()
-    ax2.plot(df['bucket'], df['stratum'], 'b-', label='Stratum')
+        ax2.plot(device_df['bucket'], device_df['stratum'], 'b-', label='Stratum')
         ax2.set_ylabel('Stratum', color='b')
         ax2.tick_params(axis='y', labelcolor='b')
-    ax2.set_yticks(range(int(df['stratum'].min()), int(df['stratum'].max()) + 1))
+        ax2.set_yticks(range(int(device_df['stratum'].min()), int(device_df['stratum'].max()) + 1))
         
-    fig.suptitle('NTP Server Numbers')
+        ax1.set_title(f'{device.capitalize()}')
+        
+        ax1.xaxis.set_major_formatter(mdates.DateFormatter('%H:%M'))
+        fig.autofmt_xdate(rotation=45)
+    
+    fig.suptitle(f'Stratum and Root Dispersion - {plot_date}')
     fig.tight_layout()
     
     img_io = io.BytesIO()
@@ -161,3 +73,57 @@ def plot_png():
 
     return Response(img_io, mimetype='image/png')
 
+
+@app.route('/ntp/packets-load.png')
+def packets_load_png():
+    dbh = psycopg.connect()
+    engine = sqlalchemy.create_engine("postgresql+psycopg://", creator=lambda: dbh)
+    query = """
+      select   time_bucket('5 minutes', time) as bucket,
+        attributes->>'Label' as device,
+        avg(cast(values->'load1'->>'value'  as float)) as load,
+        avg(cast(values->'processed-pkts'->>'value'  as int)) as packets
+      from measurements
+      where time >= date_trunc('day', now()) AND time < date_trunc('day', now()) + '1 day'::interval and
+        application = 'SNMP' and attributes->>'Label' IN ('harrison', 'david')
+      group by bucket, attributes->>'Label'
+      order by bucket, attributes->>'Label'
+    """
+    df = pd.read_sql(query, con=engine)
+    
+    
+    # Extract date for title
+    plot_date = df['bucket'].dt.date.iloc[0] if not df.empty else "Unknown Date"
+    
+    # Create figure with two side-by-side subplots
+    fig, axes = plt.subplots(1, 2, figsize=(15, 5), sharex=True)
+    
+    for i, device in enumerate(['harrison', 'david']):
+        ax1 = axes[i]
+        ax2 = ax1.twinx()
+        
+        device_df = df[df['device'] == device]
+        
+        ax1.plot(device_df['bucket'], device_df['load'], 'r-', label='CPU Load')
+        ax1.set_xlabel('Time')
+        ax1.set_ylabel('Load', color='r')
+        ax1.tick_params(axis='y', labelcolor='r')
+        
+        ax2.plot(device_df['bucket'], device_df['packets'], 'b-', label='Processed Packets')
+        ax2.set_ylabel('Packets', color='b')
+        ax2.tick_params(axis='y', labelcolor='b')
+        
+        ax1.set_title(f'{device.capitalize()}')
+        
+        ax1.xaxis.set_major_formatter(mdates.DateFormatter('%H:%M'))
+        fig.autofmt_xdate(rotation=45)
+    
+    fig.suptitle(f'CPU Load and Processed Packets - {plot_date}')
+    fig.tight_layout()
+    
+    img_io = io.BytesIO()
+    plt.savefig(img_io, format='png')
+    img_io.seek(0)
+    plt.close(fig)
+
+    return Response(img_io, mimetype='image/png')

src/pv_routes.py

@@ -12,25 +12,17 @@ from app import app
 from app import oidc
 
 
-@app.route('/pvstats')
+@app.route('/')
 @oidc.require_login
 def pvstats():
     try:
-        stepX_time = time.time()
         dbh = psycopg.connect()
         engine = sqlalchemy.create_engine("postgresql+psycopg://", creator=lambda: dbh)
 
-        step0_time = time.time()
-        df = pd.read_sql("SELECT month, cast(year AS varchar), current_energy AS value FROM pv_energy_by_month", con=engine)
-        step1_time = time.time()
-        duration1 = step1_time - step0_time
-        logger.info(f"{duration1=}")
+        df = pd.read_sql("SELECT month, cast(year AS varchar), current_energy AS value FROM pv_energy_by_month ORDER BY year, month", con=engine)
         fig_1 = px.bar(df, x='month', y='value', color='year', barmode='group')
-        step2_time = time.time()
-        duration2 = step2_time - step1_time
-        logger.info(f"{duration2=}")
         fig_1.update_layout(
-            title=f"Jahreswerte Exportierte Energie {duration1:.3f}, {duration2:.3f}",
+            title=f"Jahreswerte Exportierte Energie PV-Anlage",
             xaxis_title="",
             yaxis_title="",
             legend_title="Jahr",
@@ -43,74 +35,34 @@ def pvstats():
         )
         graph_html_1 = fig_1.to_html(full_html=False, default_height='30%')
 
-        step3_time = time.time()
-        df = pd.read_sql("SELECT time_bucket('5 minutes', time) AS bucket, AVG(power) AS avg_power FROM pv_power_v WHERE time >= date_trunc('day', now()) - '1 day'::interval AND time < date_trunc('day', now()) GROUP BY bucket ORDER BY bucket", con=engine)
-        step4_time = time.time()
-        duration3 = step4_time - step3_time
-        logger.info(f"{duration3=}")
-        fig_2 = px.line(df, x='bucket', y='avg_power')
-        step5_time = time.time()
-        duration4 = step5_time - step4_time
-        logger.info(f"{duration4=}")
+        df = pd.read_sql("SELECT month, cast(year AS varchar), current_energy AS value FROM car_energy_by_month ORDER BY year, month", con=engine)
+        fig_2 = px.bar(df, x='month', y='value', color='year', barmode='group')
         fig_2.update_layout(
+            title=f"Jahreswerte Verbrauch Elektroauto",
             xaxis_title="",
             yaxis_title="",
-            title=f"Export gestern {duration3:.3f}, {duration4:.3f}",
-            yaxis=dict(ticksuffix=" W")
+            legend_title="Jahr",
+            xaxis=dict(
+                tickmode="array",
+                tickvals=list(range(1, 13)),  # Monate 1–12
+                ticktext=["Jan", "Feb", "Mär", "Apr", "Mai", "Jun", "Jul", "Aug", "Sep", "Okt", "Nov", "Dez"]
+            ),
+            yaxis=dict(ticksuffix=" kWh")
         )
         graph_html_2 = fig_2.to_html(full_html=False, default_height='30%')
-
-        step6_time = time.time()
-        df = pd.read_sql("SELECT time_bucket('5 minutes', time) AS bucket, AVG(power) AS avg_power FROM pv_power_v WHERE time >= date_trunc('day', now()) AND time < date_trunc('day', now()) + '1 day'::interval GROUP BY bucket ORDER BY bucket", con=engine)
-        step7_time = time.time()
-        duration5 = step7_time - step6_time
-        logger.info(f"{duration5=}")
-        fig_3 = px.line(df, x='bucket', y='avg_power')
-        step8_time = time.time()
-        duration6 = step8_time - step7_time
-        logger.info(f"{duration6=}")
-        fig_3.update_layout(
-            xaxis_title="",
-            yaxis_title="",
-            title=f"Export heute {duration5:.3f}, {duration6:.3f}",
-            yaxis=dict(ticksuffix=" W")
-        )
-        graph_html_3 = fig_3.to_html(full_html=False, default_height='30%')
-        stepZ_time = time.time()
-        duration7 = stepZ_time - stepX_time
-        logger.info(f"{duration7=}")
-
         return render_template_string(f"""
             <html>
                 <head>
-                    <title>Jahreswerte PV-Energie</title>
+                    <title>Jahreswerte PV und Auto</title>
                 </head>
                 <body>
                     {graph_html_1}
                     {graph_html_2}
-                    {graph_html_3}
-                    <div style="height:9vh; background-color:lightgrey; font-family: Courier, Consolas, monospace;">
-                      <table style="border-collapse: collapse;">
-                        <style>
-                          td.smallsep {{ padding-right: 10px }}
-                          td.largesep {{ padding-right: 30px }}
-                        </style> 
-                        <tr>
-                          <td class="smallsep">Query 1:</td><td class="largesep"> {duration1:.3f} s</td><td class="smallsep">Graph 1:</td><td> {duration2:.3f} s</td>
-                        </tr><tr>
-                          <td class="smallsep">Query 2:</td><td class="largesep"> {duration3:.3f} s</td><td class="smallsep">Graph 2:</td><td> {duration4:.3f} s</td>
-                        </tr><tr>
-                          <td class="smallsep">Query 3:</td><td class="largesep"> {duration5:.3f} s</td><td class="smallsep">Graph 3:</td><td> {duration6:.3f} s</td>
-                        </tr><tr>
-                          <td class="smallsep">Total:</td><td> {duration7:.3f} s</td><td></td><td></td>
-                        </tr>
-                      </table>   
-                    </div>
                 </body>
             </html>
         """)
     except Exception as e:
-        raise Exception(f"Error when querying energy export values: {e}")
+        raise Exception(f"Error when querying energy values: {e}")
     finally:
         if dbh is not None:
             dbh.close()

src/requirements.txt

@@ -39,5 +39,5 @@ urllib3==2.3.0
 Werkzeug==3.1.3
 zipp==3.21.0
 pillow==11.1.0
-matplotlib=3.10.1
+matplotlib==3.10.1
 

src/routes.py

@@ -5,10 +5,6 @@ from app import app
 from app import oidc
 
 
-@app.route('/')
-def index():
-    abort(404)
-
 @app.route('/generate_image')
 def generate_image():
     img = Image.new('RGB', (200, 100), color=(255, 255, 255))

src/run.py

@@ -10,7 +10,7 @@ import ntp_routes
 
 
 if __name__ == '__main__':
-    app.run(port=8080)
+    app.run(host='0.0.0.0', port=8080)
 else:
     exposed_app = ProxyFix(app, x_for=1, x_host=1)