add upsert

.gitlab-ci.yml

@@ -3,12 +3,12 @@ stages:
         - dockerize
 
 variables:
-        IMAGE_NAME: registry.gitlab.com/wolutator/mosquitto-with-auth
+        IMAGE_NAME: registry.hottis.de/dockerized/mosquitto-with-auth
         HUB_IMAGE_NAME: wollud1969/mosquitto-with-auth
-        GO_BINARIES: go1.12.6.linux-amd64.tar.gz
+        GO_BINARIES: go1.19.1.linux-amd64.tar.gz
 build:
         stage: build
-        image: registry.gitlab.com/wolutator/base-build-env:latest
+        image: registry.hottis.de/dockerized/base-build-env:1.5.3-bullseye
         tags:
                 - hottis
                 - linux
@@ -22,9 +22,10 @@ build:
                   - opt/
                   - etc/
                   - generated-version.txt
+                  - version.txt
         script:
                 - apt update
-                - apt install -y gcc g++ libssl-dev uuid-dev xsltproc docbook docbook-xsl libmariadbclient-dev libwebsockets-dev
+                - apt install -y gcc g++ libssl-dev uuid-dev libcjson-dev xsltproc docbook docbook-xsl libmariadb-dev libpq-dev libwebsockets-dev
                 - wget https://dl.google.com/go/$GO_BINARIES
                 - tar -xzf $GO_BINARIES
                 - mv go /usr/local
@@ -46,6 +47,7 @@ build:
                 - cp pw $BUILD_DIR/opt/bin
                 - popd
                 - VERSION=`cat VERSION`
+                - echo -n "$VERSION" > version.txt
                 - REFCNT=`git rev-list --all --count`
                 - echo -n "$VERSION.$REFCNT.$CI_COMMIT_REF_NAME" > generated-version.txt 
 
@@ -59,12 +61,16 @@ dockerize:
         dependencies:
                 - build
         script:
-                - VERSION=`cat generated-version.txt`
+                - GENERATED_VERSION=`cat generated-version.txt`
-                - docker build --tag $IMAGE_NAME:latest --tag $IMAGE_NAME:$VERSION --tag $HUB_IMAGE_NAME:latest --tag $HUB_IMAGE_NAME:$VERSION .
+                - VERSION=`cat version.txt`
+                - echo docker build --tag $IMAGE_NAME:latest --tag $IMAGE_NAME:$VERSION --tag $IMAGE_NAME:$GENERATED_VERSION --tag $HUB_IMAGE_NAME:latest --tag $HUB_IMAGE_NAME:$VERSION --tag $HUB_IMAGE_NAME:$GENERATED_VERSION .
+                - docker build --tag $IMAGE_NAME:latest --tag $IMAGE_NAME:$VERSION --tag $IMAGE_NAME:$GENERATED_VERSION --tag $HUB_IMAGE_NAME:latest --tag $HUB_IMAGE_NAME:$VERSION --tag $HUB_IMAGE_NAME:$GENERATED_VERSION .
                 - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
                 - docker push $IMAGE_NAME:latest
                 - docker push $IMAGE_NAME:$VERSION
+                - docker push $IMAGE_NAME:$GENERATED_VERSION
                 - docker login -u $DOCKER_HUB_LOGIN -p $DOCKER_HUB_PASSWORD
                 - docker push $HUB_IMAGE_NAME:latest
                 - docker push $HUB_IMAGE_NAME:$VERSION
+                - docker push $HUB_IMAGE_NAME:$GENERATED_VERSION
 

.gitmodules

@@ -3,4 +3,4 @@
 	url = https://github.com/eclipse/mosquitto.git
 [submodule "parts/mosquitto-go-auth"]
 	path = parts/mosquitto-go-auth
-	url = https://github.com/wollud1969/mosquitto-go-auth.git
+	url = https://github.com/iegomez/mosquitto-go-auth.git

Dockerfile

@@ -1,7 +1,7 @@
-FROM debian:latest
+FROM debian:bullseye
 
 LABEL Maintainer="Wolfgang Hottgenroth <woho@hottis.de>"
-LABEL ImageName="registry.gitlab.com/wolutator/mosquitto-with-auth"
+LABEL ImageName="registry.hottis.de/dockerized/mosquitto-with-auth"
 LABEL AlternativeImageName="wollud1969/mosquitto-with-auth"
 
 ARG MOSQ_USER="mosquitto"
@@ -10,7 +10,9 @@ ARG MOSQ_GID="1883"
 
 RUN \
   apt update && \
-  apt install -y mariadb-client openssl libwebsockets8 && \
+  apt install -y mariadb-client libpq5 openssl libwebsockets-dev certbot bash cron supervisor vim-tiny procps net-tools && \
+  update-alternatives --set editor /usr/bin/vim.tiny && \
+  update-alternatives --set vi /usr/bin/vim.tiny && \
   groupadd -r -g $MOSQ_GID $MOSQ_USER && \
   useradd -m -r -u $MOSQ_UID -g $MOSQ_USER $MOSQ_USER && \
   mkdir -p /opt/data  
@@ -18,14 +20,24 @@ RUN \
 
 COPY opt/ /opt
 COPY etc/ /opt/etc
+COPY supervisor-mosquitto.conf /etc/supervisor/conf.d/
+COPY crontab /etc/
+COPY mosquitto.conf-sample /opt/etc/mosquitto/
+COPY cert-deploy.sh /opt/bin/
 
 VOLUME /opt/etc
 VOLUME /opt/data
+VOLUME /var/log/supervisor
+VOLUME /etc/letsencrypt
+
+EXPOSE 80/TCP
+EXPOSE 443/TCP
 EXPOSE 1883/tcp
 EXPOSE 8883/tcp
 EXPOSE 9001/tcp
 
 WORKDIR /opt
 
-CMD ["env", "LD_LIBRARY_PATH=/opt/lib", "/opt/sbin/mosquitto", "-c", "/opt/etc/mosquitto/mosquitto.conf"]
+CMD [ "/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf" ]
+
 

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019 Wolfgang Hottgenroth
+Copyright (c) 2019, 2022 Wolfgang Hottgenroth
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

VERSION

@@ -1 +1 @@
-0.1
+2.0.15-2.0.0-01

cert-deploy.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+MY_DOMAIN=example.com
+CERTIFICATE_DIR=/opt/etc/mosquitto/
+
+if [ "${RENEWED_DOMAINS}" = "${MY_DOMAIN}" ]; then
+	cp ${RENEWED_LINEAGE}/fullchain.pem ${CERTIFICATE_DIR}/server.crt
+	cp ${RENEWED_LINEAGE}/privkey.pem ${CERTIFICATE_DIR}/server.key
+	chown mosquitto: ${CERTIFICATE_DIR}/server.crt ${CERTIFICATE_DIR}/server.key
+	chmod 0600 ${CERTIFICATE_DIR}/server.crt ${CERTIFICATE_DIR}/server.key
+    supervisorctl restart mosquitto
+fi

create-schema-mariadb.sql

@@ -15,9 +15,10 @@ CREATE TABLE acls_t (
     id INTEGER AUTO_INCREMENT,
     user INTEGER NOT NULL,
     topic VARCHAR(256) NOT NULL,
-    rw INTEGER(1) NOT NULL DEFAULT 1,	-- 1 is read, 2 is write, 3 is readwrite, 4 is subscribe
+    -- rw, bitmask: 1 is read, 2 is write, 3 is readwrite, 4 is subscribe
+    rw INTEGER(1) NOT NULL DEFAULT 1,	
     PRIMARY KEY (id),
-    CONSTRAINT `fk_book_author`
+    CONSTRAINT `fk_user_acl`
       FOREIGN KEY (user) REFERENCES users_t (id)
       ON DELETE CASCADE
       ON UPDATE CASCADE

create-schema-postgres.sql

@@ -0,0 +1,35 @@
+CREATE TABLE public.users_t (
+    id SERIAL NOT NULL,
+    username VARCHAR(25) NOT NULL,
+    pw VARCHAR(512) NOT NULL,
+    super INTEGER DEFAULT 0 NOT NULL,
+    CONSTRAINT users_t_pk PRIMARY KEY (id),
+    CONSTRAINT users_t_uk_username UNIQUE (username)
+);
+
+CREATE TABLE public.acls_t (
+    id SERIAL NOT NULL,
+    "user" INTEGER NOT NULL,
+    topic VARCHAR(512) NOT NULL,
+    rw INTEGER DEFAULT 5 NOT NULL,
+    CONSTRAINT acls_t_pk PRIMARY KEY (id),
+    CONSTRAINT acls_t_fk_user FOREIGN KEY ("user") REFERENCES users_t(id)
+);
+
+CREATE OR REPLACE VIEW users AS
+    SELECT users_t.username,
+           users_t.pw,
+           users_t.super
+      FROM users_t;
+
+CREATE OR REPLACE VIEW acls AS
+    SELECT a.topic,
+           a.rw,
+           u.username
+      FROM users_t u,
+           acls_t a
+      WHERE a."user" = u.id;
+
+CREATE USER mosquittoauth;
+GRANT SELECT ON users, acls TO mosquittoauth;
+

crontab

@@ -0,0 +1,3 @@
+SHELL=/bin/sh
+PATH=/usr/bin
+1 1 * * 1 root supervisorctl start certbot

hivemq-to-mosquitto-auth-view.sql

@@ -0,0 +1,5 @@
+create or replace view hivemq_to_mosquitto_auth_v as
+  select username,
+         'PBKDF2$' || lower(algorithm) || '$' || password_iterations || '$' || password_salt || '$' || "password" as pw
+    from users;
+         

mosquitto-start.sh

@@ -1,23 +1,30 @@
 #!/bin/bash
 
-IMAGE=registry.gitlab.com/wolutator/mosquitto-with-auth:latest
+IMAGE=wollud1969/mosquitto-with-auth:latest
 VOLUME_CONFIG=mosquitto-config
 VOLUME_DATA=mosquitto-data
+VOLUME_LOG=mosquitto-log
+VOLUME_LETSENCRYPT=mosquitto-letsencrypt
 
 docker volume inspect $VOLUME_CONFIG > /dev/null || docker volume create $VOLUME_CONFIG
 docker volume inspect $VOLUME_DATA > /dev/null || docker volume create $VOLUME_DATA
+docker volume inspect $VOLUME_LOG > /dev/null || docker volume create $VOLUME_LOG
+docker volume inspect $VOLUME_LETSENCRYPT > /dev/null || docker volume create $VOLUME_LETSENCRYPT
 
 docker pull $IMAGE
 
 docker run \
   -d \
   --rm \
+  -p80:80 \
+  -p443:443 \
   -p1883:1883 \
   -p8883:8883 \
   -p9001:9001 \
   -v $VOLUME_CONFIG:/opt/etc/mosquitto \
   -v $VOLUME_DATA:/opt/data \
-  --link mariadb \
+  -v $VOLUME_LOG:/var/log/supervisor \
+  -v $VOLUME_LETSENCRYPT:/etc/letsencrypt \
   --name mosquitto \
   $IMAGE
 

mosquitto.conf-sample

@@ -8,6 +8,15 @@ protocol mqtt
 #allow_anonymous true
 allow_anonymous false
 
+listener 8883
+protocol mqtt
+#allow_anonymous true
+allow_anonymous false
+certfile /opt/etc/mosquitto/server.crt
+keyfile /opt/etc/mosquitto/server.key
+dhparamfile /opt/etc/mosquitto/dh.pem
+tls_version tlsv1.2
+
 auth_plugin /opt/lib/go-auth.so
 auth_opt_log_dest stdout
 auth_opt_log_level debug

readme.md

@@ -2,6 +2,8 @@
 
 This project includes the mosquitto MQTT broker (https://github.com/eclipse/mosquitto, see also https://mosquitto.org/) and the mosquitto-go-auth (https://github.com/iegomez/mosquitto-go-auth forked into https://github.com/wollud1969/mosquitto-go-auth) as submodules.
 
+It additionally includes the Let's Encrypt `certbot` and some mimic for automatic renewal of certificates using `supervisord` and `cron`.
+
 Using Gitlab CI and a Dockerfile included in this project a Docker image based on Debian Linux is created.
 
 
@@ -17,41 +19,53 @@ The mosquitto-go-auth supports a couple of backends and it seems that all backen
 
 ## Running the container
 
+You can not run a container based on this image "out-of-the-box". You need to edit the configuration, and if desired, run all the Let's Encrypt stuff. For details see below.
+
 The container exposed the ports 1883 (MQTT), 8883 (MQTT over SSL) and 9001 (MQTT over websockets). Only the configuration directory containing `mosquitto.conf` and friends is prepared as a volume.
 
-All logging is send to `stdout`, so it can be inspected using `docker logs -f <mosquitto-container>`
+Besides the mosquitto configuration volume, there are volume required for the Let's Encrypt configuration and state, the data directory of the broker and for the logfiles for `supervisord`.
+
+Due to the requirements of `certbot` it also exposed the port 80 and 443. So, be careful when trying to start this image as a container on the same host as a webserver.
+
+All logging is send into a dedicated logfile under control of `supervisord`.
 
 To start the container a script is provided, which might need to adjusted to the actual environment:
 
 
     #!/bin/bash
 
-    IMAGE=registry.gitlab.com/wolutator/mosquitto-with-auth:latest
+    IMAGE=wollud1969/mosquitto-with-auth:latest
     VOLUME_CONFIG=mosquitto-config
     VOLUME_DATA=mosquitto-data
+    VOLUME_LOG=mosquitto-log
+    VOLUME_LETSENCRYPT=mosquitto-letsencrypt
 
     docker volume inspect $VOLUME_CONFIG > /dev/null || docker volume create $VOLUME_CONFIG
     docker volume inspect $VOLUME_DATA > /dev/null || docker volume create $VOLUME_DATA
+    docker volume inspect $VOLUME_LOG > /dev/null || docker volume create $VOLUME_LOG
+    docker volume inspect $VOLUME_LETSENCRYPT > /dev/null || docker volume create $VOLUME_LETSENCRYPT
 
     docker pull $IMAGE
 
     docker run \
-        -d \
+      -d \
-        --rm \
+      --rm \
-        -p1883:1883 \
+      -p80:80 \
-        -p8883:8883 \
+      -p443:443 \
-        -p9001:9001 \
+      -p1883:1883 \
-        -v $VOLUME_CONFIG:/opt/etc/mosquitto \
+      -p8883:8883 \
-        -v $VOLUME_DATA:/opt/data \
+      -p9001:9001 \
-        --link mariadb \
+      -v $VOLUME_CONFIG:/opt/etc/mosquitto \
-        --name mosquitto \
+      -v $VOLUME_DATA:/opt/data \
-        $IMAGE
+      -v $VOLUME_LOG:/var/log/supervisor \
+      -v $VOLUME_LETSENCRYPT:/etc/letsencrypt \
+      --name mosquitto \
+      $IMAGE
 
 
+The container expects the main configuration file in the root of the configuration volume named `mosquitto.conf`.
 
-The container expects the main configuration file in the root of the volume named `mosquitto.conf`.
+A very simple configuration, supporting MQTT on port 1883 and over TLS on port 8883 is:
-
-A very simple configuration, only supporting MQTT on port 1883 is:
 
 
     log_dest stdout
@@ -64,6 +78,15 @@ A very simple configuration, only supporting MQTT on port 1883 is:
     #allow_anonymous true
     allow_anonymous false
 
+    listener 8883
+    protocol mqtt
+    #allow_anonymous true
+    allow_anonymous false
+    certfile /opt/etc/mosquitto/server.crt
+    keyfile /opt/etc/mosquitto/server.key
+    dhparamfile /opt/etc/mosquitto/dh.pem
+    tls_version tlsv1.2
+
     auth_plugin /opt/lib/go-auth.so
     auth_opt_log_dest stdout
     auth_opt_log_level debug
@@ -104,7 +127,7 @@ The required schema in the database is
 	    topic VARCHAR(256) NOT NULL,
 	    rw INTEGER(1) NOT NULL DEFAULT 1,	-- 1 is read, 2 is write, 3 is readwrite, 4 is subscribe
 	    PRIMARY KEY (id),
-	    CONSTRAINT `fk_book_author`
+	    CONSTRAINT `fk_users_user`
 	      FOREIGN KEY (user) REFERENCES users_t (id)
 	      ON DELETE CASCADE
 	      ON UPDATE CASCADE
@@ -122,6 +145,26 @@ The password is generated using the `pw` tool provided by mosquitto-go-auth, whi
 
 For further information consult the readme and the examples in the mosquitto-go-auth project (https://github.com/iegomez/mosquitto-go-auth or https://github.com/wollud1969/mosquitto-go-auth).
 
+For MariaDB and PostgreSQL there are prepared table create statements in the repository,
+
+For PostgresSQL there is a prepared Python tool in the directory `tools` available to added users into the database.
 
 
+## Preparing configuration and certificates
+
+* Start the container using the provided start script, follow the container log using `docker logs -f <containername>`, you will see that `supervisord` start `cron` and `mosquitto` and you will see that the start of `mosquitto` fails
+* Go into the container using `docker exec -it <containername> bash`
+* Go into the directory `/opt/etc/mosquitto`, copy `mosquitto.conf-sample` into `mosquitto.conf` and edit it if required
+
+If  you want to register at Let's Encrypt and obtain a certificate follow the next steps:
+
+* Generate Diffie-Hellman parameters in the broker's configuration directory using `openssl dhparam -out /opt/etc/mosquitto/dh.pem 2048`
+* Register at Let's Encrypt using `certbot register`
+* Obtain a certificate using `certbot certonly -d <domainname> --standalone`, make sure to add the domainname into DNS first
+* Copy the deployment script into the deploy hooks directory of Let's Encrypt: `cp /opt/bin/cert-deploy.sh /etc/letsencrypt/renewal-hooks/deploy/`, edit it to fill in the right domainname
+* Run the deployment script manually for the very first deployment of certificates: `env RENEWED_DOMAINS=<domainname> RENEWED_LINEAGE=/etc/letsencrypt/live/<domainname> ./cert-deploy.sh`
+* The certificate and private key is now copied from the Let's Encrypt state directory into the configuration directory of `mosquitto` and the broker is restarted, you can observe that in the container logging output
+* Finally, test the broker using something like `mosquitto_sub -h <domainname> -p 8883 --tls-version tlsv1.2 -v -t test` and `mosquitto_pub -h <domainname> -p 8883 --tls-version tlsv1.2 -t test -m test123`
+* Renewal of the certificate will be triggered once a week
+
 

supervisor-mosquitto.conf

@@ -0,0 +1,21 @@
+[supervisord]
+nodaemon=true
+user=root
+
+[program:mosquitto]
+environment=LD_LIBRARY_PATH="/opt/lib"
+command=/opt/sbin/mosquitto -c /opt/etc/mosquitto/mosquitto.conf
+autostart=true
+autorestart=true
+
+[program:certbot]
+command=/usr/bin/certbot renew --standalone
+autostart=false
+autorestart=false
+startsecs=0
+
+[program:cron]
+command=/usr/sbin/cron -f
+autostart=true
+autorestart=true
+

tools/.gitignore

@@ -0,0 +1 @@
+.venv/

tools/genpw.py

@@ -0,0 +1,89 @@
+
+from pbkdf2 import PBKDF2
+from hashlib import sha512
+from base64 import b64encode
+import argparse
+import secrets
+import string
+import psycopg2
+
+
+parser = argparse.ArgumentParser(description='genpw')
+parser.add_argument('--length', '-l',
+                    help='Length of auto-generated password', 
+                    default='24',
+                    required=False)
+parser.add_argument('--password', '-p',
+                    help='Password', 
+                    required=False)
+parser.add_argument('--username', '-u',
+                    help='Username',
+                    required=False)
+parser.add_argument('--topic', '-t',
+                    help='Initially granted topic',
+                    required=False)
+parser.add_argument('--acl', '-a',
+                    help='ACL value for topic, Bit0=read, Bit1=write, Bit2=subscribe',
+                    required=False)
+parser.add_argument('--printonly', '-o',
+                    help='Just print the password hash, do not write to database',
+                    action='store_true')
+
+args = parser.parse_args()
+length = args.length
+password = args.password
+
+print_only = args.printonly
+
+alphabet = string.ascii_letters + string.digits
+iterations = 100000
+
+if (not password):
+    if (not length):
+        raise Exception("Either length or password must be given")
+    password = ''.join(secrets.choice(alphabet) for i in range(int(length)))
+
+salt = secrets.token_bytes(16)
+hash = b64encode(PBKDF2(password, salt, iterations=iterations, digestmodule=sha512).read(64)).decode()
+
+salt_b64 = b64encode(salt).decode()
+
+pw = f"PBKDF2$sha512${iterations}${salt_b64}${hash}"
+print(f"{password=}")
+print(f"hash={pw}")
+
+if not print_only:
+    login = args.username
+    if (not login):
+        raise Exception("For writing to database a username must be given")
+    topic = args.topic
+    acl = args.acl
+
+    conn = psycopg2.connect()
+    conn.autocommit = False
+
+    try:
+        with conn:
+            with conn.cursor() as cur:
+                cur.execute("""
+                            insert into users_t (username, pw) 
+                                values(%(username)s, %(pw)s) 
+                                on conflict on constraint users_t_uk_username 
+                                    do update set pw = %(pw)s
+                                returning id 
+                            """,
+                            { 'username': login, 'pw': pw })
+                res = cur.fetchone()
+                if res is None:
+                    raise Exception("Unable to add user to database")
+                id = res[0]
+                print("User added to database")
+            if (topic and acl):
+                acl = int(acl)
+                with conn.cursor() as cur:
+                    cur.execute('insert into acls_t ("user", topic, rw) values(%(user)s, %(topic)s, %(rw)s)',
+                                { 'user': id, 'topic': topic, 'rw': acl })
+                    print("ACL added to database")
+    finally:
+        if conn:
+            conn.close()

tools/requirements.txt

@@ -0,0 +1,2 @@
+pbkdf2==1.3
+psycopg2==2.9.5