remove pickle imports

snippets/test8.py

@@ -0,0 +1,105 @@
+import unittest
+import json
+
+class A(object):
+    def __init__(self, arg1=None, arg2=None):
+        self.arg1 = arg1
+        self.arg2 = arg2
+        self.arg3 = self.arg1 + self.arg2
+
+    def __str__(self):
+        return "A: {0!s} {1!s} {2!s}".format(self.arg1, self.arg2, self.arg3)
+
+    def jsonify(self):
+        return {'type':self.__class__.__name__, 'args': {'arg1':self.arg1, 'arg2':self.arg2}}
+
+class MyEncoder(json.JSONEncoder):
+    def default(self, o):
+        try:
+            return o.jsonify()
+        except TypeError or AttributeError:
+            return super().default(o)
+
+
+def objectFactory(j):
+    klass = eval(j['type'])
+    o = klass(**j['args'])
+    return o
+
+def MyDecoder(j):
+    if type(j) == dict and 'type' in j:
+        return objectFactory(j)
+    else:
+        return j
+
+class Tests(unittest.TestCase):
+    def test_a1(self):
+        a1 = A(1, 2)
+        self.assertEqual(a1.arg1, 1)
+        self.assertEqual(a1.arg2, 2)
+        self.assertEqual(a1.arg3, 3)
+
+    def test_a2(self):
+        a2 = A(**{'arg1':2, 'arg2':4})
+        self.assertEqual(a2.arg1, 2)
+        self.assertEqual(a2.arg2, 4)
+        self.assertEqual(a2.arg3, 6)
+
+    def test_a3(self):
+        j = '{ "type": "A", "args": { "arg1": 3, "arg2": 5 } }'
+        jj = json.loads(j)
+        klass = eval(jj['type'])
+        self.assertEqual(A, klass)
+        a3 = klass(**jj['args'])
+        self.assertEqual(a3.arg1, 3)
+        self.assertEqual(a3.arg2, 5)
+        self.assertEqual(a3.arg3, 8)
+
+    def test_a4(self):
+        j = '{ "type": "A", "args": { "arg1": 3, "arg2": 5 } }'
+        jj = json.loads(j)
+        klass = eval(jj['type'])
+        self.assertEqual(A, klass)
+        a3 = klass(**jj['args'])
+        self.assertEqual(a3.arg1, 3)
+        self.assertEqual(a3.arg2, 5)
+        self.assertEqual(a3.arg3, 8)
+
+        jjjj = json.dumps(a3, cls=MyEncoder)
+
+        jjj = json.loads(jjjj)
+        klass = eval(jjj['type'])
+        self.assertEqual(A, klass)
+        a3 = klass(**jjj['args'])
+        self.assertEqual(a3.arg1, 3)
+        self.assertEqual(a3.arg2, 5)
+        self.assertEqual(a3.arg3, 8)
+
+    def test_a5(self):
+        jList = []
+        jList.append(objectFactory({'type':'A', 'args': {'arg1':1, 'arg2':2}}))
+        jList.append(objectFactory({'type':'A', 'args': {'arg1':2, 'arg2':3}}))
+        jList.append(objectFactory({'type':'A', 'args': {'arg1':3, 'arg2':4}}))
+
+        js = json.dumps(jList, cls=MyEncoder, sort_keys=True, indent=4)
+
+        print(js)
+
+        jResultList = json.loads(js, object_hook=MyDecoder)
+
+        self.assertEqual(jResultList[0].arg1, 1)
+        self.assertEqual(jResultList[0].arg2, 2)
+        self.assertEqual(jResultList[0].arg3, 3)
+
+        self.assertEqual(jResultList[1].arg1, 2)
+        self.assertEqual(jResultList[1].arg2, 3)
+        self.assertEqual(jResultList[1].arg3, 5)
+
+        self.assertEqual(jResultList[2].arg1, 3)
+        self.assertEqual(jResultList[2].arg2, 4)
+        self.assertEqual(jResultList[2].arg3, 7)
+
+
+
+if __name__ == '__main__':
+    unittest.main()

snippets/test9.py

@@ -0,0 +1,14 @@
+class A(object):
+    def __init__(self):
+        self.a = 1
+    def x(self):
+        return self.a
+
+class B(A):
+    def __init__(self):
+        self.a = 2
+    def x(self):
+        return self.a
+    def y(self):
+        return super().x()
+

src/CmdServer.py

@@ -4,9 +4,8 @@ import cmd
 import re
 import io
 import datetime
-import pickle
-import sys
 import RegisterDatapoint
+import logging
 
 class CmdInterpreterException(ValueError): pass
 
@@ -30,6 +29,7 @@ class CmdInterpreter(cmd.Cmd):
         self.prompt = "test8> "
         self.intro = "test8 admin interface"
         self.splitterRe = re.compile('\s+')
+        self.logger = logging.getLogger('CmdInterpreter')
 
     def __print(self, text):
         self.stdout.write(text)
@@ -307,45 +307,35 @@ class CmdInterpreter(cmd.Cmd):
         self.__println("DO NOT FORGET TO SAVE AFTERWARDS!")
 
     def do_save(self, arg):
-        with open(self.config.registerFile, 'wb') as f:
+        RegisterDatapoint.saveRegisterList(self.registers, self.config.registerFile)
-            pickle.dump(self.registers, f)
 
     def help_save(self):
         self.__println("Usage: save")
         self.__println("Saves a modified register list into the register file.")
 
     def do_load(self, arg):
-        registers = None
-        with open(self.config.registerFile, 'rb') as f:
-            registers = pickle.load(f)
         try:
-            RegisterDatapoint.checkRegisterList(registers)
+            registers = RegisterDatapoint.loadRegisterList(self.config.registerFile)
             self.registers = registers
-        except ValueError as e:
+        except Exception as e:
             self.__println("Unable to load register list: {0!s}".format(e))
 
     def help_load(self):
         self.__println("Usage: load")
         self.__println("Reload the register file, overwrite all unsaved changes.")
 
-    #def do_shutdown(self, arg):
-    #    sys.exit()
-
-    #def help_shutdown(self):
-    #    self.__println("Usage: shutdown")
-    #    self.__println("Shuts down the application")
-
 
 
 class CmdHandle(socketserver.StreamRequestHandler):
     def handle(self):
+        logger = logging.getLogger('CmdHandle')
         cmd = CmdInterpreter(io.TextIOWrapper(self.rfile), io.TextIOWrapper(self.wfile), self.server.userData.config, 
                              self.server.userData.notifier, self.server.userData.registers)
         try:
             cmd.cmdloop()
-            print("Cmd handle terminated")
+            logger.info("Cmd handle terminated")
         except ConnectionAbortedError as e:
-            print("Cmd handle externally interrupted")
+            logger.info("Cmd handle externally interrupted")
 
 class MyThreadingTCPServer(socketserver.ThreadingTCPServer):
     def __init__(self, host, handler, userData):

src/Config.py

@@ -9,7 +9,7 @@ class Config(object):
         self.mqttPassword = ''
         self.cmdAddress = '127.0.0.1'
         self.cmdPort = 9999
-        self.registerFile = 'registers.pkl'
+        self.registerFile = 'registers.json'
         self.serialPort = '/dev/ttyAMA0'
         self.serialBaudRate = 9600
         self.interCommDelay = 0.025

src/MqttProcessor.py

@@ -34,7 +34,7 @@ class MqttProcessor(threading.Thread, AbstractNotificationReceiver):
     def __processUpdatedRegisters(self, force=False):
         self.logger.debug("MqttProcessor.__updateSubscriptions")
 
-        subscribeTopics = [ r.subscribeTopic for r in self.registers if r.subscribeTopic]
+        subscribeTopics = [ r.subscribeTopic for r in self.registers if hasattr(r,'subscribeTopic') and r.subscribeTopic]
         self.logger.debug("Topics: {0!s}".format(subscribeTopics))
 
         for subscribeTopic in subscribeTopics:
@@ -49,7 +49,7 @@ class MqttProcessor(threading.Thread, AbstractNotificationReceiver):
                 self.client.unsubscribe(subscription)
                 self.subscriptions.remove(subscription)
 
-        self.topicRegisterMap = { r.subscribeTopic: r for r in self.registers if r.subscribeTopic }
+        self.topicRegisterMap = { r.subscribeTopic: r for r in self.registers if hasattr(r,'subscribeTopic') and r.subscribeTopic }
 
     def receiveNotification(self, arg):
         self.logger.info("MqttProcessor:registersChanged")

src/RegisterDatapoint.py

@@ -3,18 +3,44 @@ from pymodbus.pdu import ExceptionResponse
 from pymodbus.exceptions import ModbusIOException
 import MqttProcessor
 import logging
-import pickle
+import json
+
+
+class JsonifyEncoder(json.JSONEncoder):
+    def default(self, o):
+        res = None
+        try:
+            res = o.jsonify()
+        except (TypeError, AttributeError):
+            if type(o) == datetime.timedelta:
+                res = o.total_seconds()
+            else:
+                res = super().default(o)
+        return res
+
+def datapointObjectHook(j):
+    if type(j) == dict and 'type' in j and 'args' in j:
+        klass = eval(j['type'])
+        o = klass(**j['args'])
+        return o
+    else:
+        return j
+
 
 
 class DatapointException(Exception): pass
 
 class AbstractModbusDatapoint(object):
     def __init__(self, label=None, unit=None, address=None, count=None, scanRate=None):
+        self.argList = ['label', 'unit', 'address', 'count', 'scanRate']
         self.label = label
         self.unit = unit
         self.address = address
         self.count = count
-        self.scanRate = scanRate
+        if type(scanRate) == float:
+            self.scanRate = datetime.timedelta(seconds=scanRate)
+        else:
+            self.scanRate = scanRate
         self.type = 'abstract data point'
         self.enqueued = False
         self.lastContact = None
@@ -32,14 +58,21 @@ class AbstractModbusDatapoint(object):
                         self.scanRate, self.enqueued, self.lastContact,
                         self.errorCount, self.processCount))
 
+    def jsonify(self):
+        return {'type':self.__class__.__name__, 
+                'args': { k: getattr(self, k) for k in self.argList }
+               }
+
     def process(self, client):
         raise NotImplementedError
 
 
+
 class HoldingRegisterDatapoint(AbstractModbusDatapoint):
     def __init__(self, label=None, unit=None, address=None, count=None, scanRate=None, 
                  publishTopic=None, subscribeTopic=None, feedbackTopic=None):
         super().__init__(label, unit, address, count, scanRate)
+        self.argList = self.argList + ['publishTopic', 'subscribeTopic', 'feedbackTopic']
         self.publishTopic = publishTopic
         self.subscribeTopic = subscribeTopic
         self.feedbackTopic = feedbackTopic
@@ -53,14 +86,14 @@ class HoldingRegisterDatapoint(AbstractModbusDatapoint):
                         self.writeRequestValue))
 
     def process(self, client, pubQueue):
-        self.logger = logging.getLogger('HoldingRegisterDatapoint')
+        logger = logging.getLogger('HoldingRegisterDatapoint')
         if self.writeRequestValue:
             # perform write operation
-            self.logger.debug("Holding register, perform write operation")
+            logger.debug("Holding register, perform write operation")
             self.writeRequestValue = None
         else:
             # perform read operation
-            self.logger.debug("Holding register, perform read operation")
+            logger.debug("Holding register, perform read operation")
             self.processCount += 1
             result = client.read_holding_registers(address=self.address, 
                                                    count=self.count, 
@@ -68,7 +101,7 @@ class HoldingRegisterDatapoint(AbstractModbusDatapoint):
             if type(result) in [ExceptionResponse, ModbusIOException]:
                 self.errorCount += 1
                 raise DatapointException(result)
-            self.logger.debug("{0}: {1!s}".format(self.label, result.registers))
+            logger.debug("{0}: {1!s}".format(self.label, result.registers))
             pubQueue.put(MqttProcessor.PublishItem(self.publishTopic, str(result.registers)))
             self.lastContact = datetime.datetime.now()
     
@@ -79,6 +112,7 @@ class HoldingRegisterDatapoint(AbstractModbusDatapoint):
 class ReadOnlyDatapoint(AbstractModbusDatapoint):
     def __init__(self, label=None, unit=None, address=None, count=None, scanRate=None, updateOnly=None, publishTopic=None):
         super().__init__(label, unit, address, count, scanRate)
+        self.argList = self.argList + ['updateOnly', 'publishTopic']
         self.updateOnly = updateOnly
         self.lastValue = None
         self.publishTopic = publishTopic
@@ -96,9 +130,9 @@ class InputRegisterDatapoint(ReadOnlyDatapoint):
         self.type = 'input register'
 
     def process(self, client, pubQueue):
-        self.logger = logging.getLogger('InputRegisterDatapoint')
+        logger = logging.getLogger('InputRegisterDatapoint')
         # perform read operation
-        self.logger.debug("Input register, perform read operation")
+        logger.debug("Input register, perform read operation")
         self.processCount += 1
         result = client.read_input_registers(address=self.address,
                                              count=self.count,
@@ -108,7 +142,7 @@ class InputRegisterDatapoint(ReadOnlyDatapoint):
             raise DatapointException(result)
         if not self.updateOnly or (result.registers != self.lastValue):
             self.lastValue = result.registers
-            self.logger.debug("{0}: {1!s}".format(self.label, result.registers))        
+            logger.debug("{0}: {1!s}".format(self.label, result.registers))        
             pubQueue.put(MqttProcessor.PublishItem(self.publishTopic, str(result.registers)))
         self.lastContact = datetime.datetime.now()
 
@@ -119,9 +153,9 @@ class DiscreteInputDatapoint(ReadOnlyDatapoint):
         self.type = 'discrete input'
 
     def process(self, client, pubQueue):
-        self.logger = logging.getLogger('DiscreteInputDatapoint')
+        logger = logging.getLogger('DiscreteInputDatapoint')
         # perform read operation
-        self.logger.debug("Discrete input, perform read operation")
+        logger.debug("Discrete input, perform read operation")
         self.processCount += 1
         result = client.read_discrete_inputs(address=self.address,
                                              count=self.count,
@@ -131,45 +165,20 @@ class DiscreteInputDatapoint(ReadOnlyDatapoint):
             raise DatapointException(result)
         if not self.updateOnly or (result.bits != self.lastValue):
             self.lastValue = result.bits
-            self.logger.debug("{0}: {1!s}".format(self.label, result.bits))        
+            logger.debug("{0}: {1!s}".format(self.label, result.bits))        
             pubQueue.put(MqttProcessor.PublishItem(self.publishTopic, str(result.bits)))
         self.lastContact = datetime.datetime.now()
 
 
 
-def loadRegisterList(registerList):
+def saveRegisterList(registerList, registerListFile):
-    # Load, check and auto-update registers file
+    js = json.dumps(registerList, cls=JsonifyEncoder, sort_keys=True, indent=4)
-
+    with open(registerListFile, 'w') as f:
-    with open(registerList, 'rb') as f:
+        f.write(js)
-        datapoints = pickle.load(f)
-
-    checkRegisterList(datapoints)
-
-    newDatapoints = []
-    for dp in datapoints:
-        ndp = type(dp)()
-        for k,v in dp.__dict__.items():
-            ndp.__dict__[k] = v
-        newDatapoints.append(ndp)
-        logging.getLogger('loadRegisterList').debug("Datapoint loaded: {0!s}".format(ndp))
-
-    checkRegisterList(newDatapoints, reset=True)
-
-    with open(registerList, 'wb') as f:
-        pickle.dump(newDatapoints, f)
-
-    return newDatapoints
-
-
-def checkRegisterList(registers, reset=False):
-    for r in registers:
-        if not isinstance(r, AbstractModbusDatapoint):
-            raise ValueError('Entry in register list {0!s} is not derived from class AbstractModbusDatapoint'.format(r))        
-        else:
-            if reset:
-                r.errorCount = 0
-                r.processCount = 0
-                r.enqueued = False
-
     
+def loadRegisterList(registerListFile):
+    with open(registerListFile, 'r') as f:
+        js = f.read()
+        registerList = json.loads(js, object_hook=datapointObjectHook)
+        return registerList
 

src/loadRegisterFile.py

@@ -0,0 +1,6 @@
+import RegisterDatapoint
+
+registers = RegisterDatapoint.loadRegisterList('registers.json')
+
+for r in registers:
+    print("{0!s}".format(r))

src/registers.json

@@ -0,0 +1,129 @@
+[
+    {
+        "args": {
+            "address": 1,
+            "count": 1,
+            "label": "Temperature",
+            "publishTopic": "Pub/Temperature",
+            "scanRate": 60.0,
+            "unit": 5,
+            "updateOnly": false
+        },
+        "type": "InputRegisterDatapoint"
+    },
+    {
+        "args": {
+            "address": 2,
+            "count": 1,
+            "label": "Humidity",
+            "publishTopic": "Pub/Humidity",
+            "scanRate": null,
+            "unit": 5,
+            "updateOnly": false
+        },
+        "type": "InputRegisterDatapoint"
+    },
+    {
+        "args": {
+            "address": 0,
+            "count": 1,
+            "label": "Switches",
+            "publishTopic": "Pub/Switches",
+            "scanRate": 0.25,
+            "unit": 4,
+            "updateOnly": false
+        },
+        "type": "DiscreteInputDatapoint"
+    },
+    {
+        "args": {
+            "address": 40010,
+            "count": 2,
+            "feedbackTopic": null,
+            "label": "Counter1",
+            "publishTopic": "Pub/Counter1",
+            "scanRate": 60.0,
+            "subscribeTopic": null,
+            "unit": 4
+        },
+        "type": "HoldingRegisterDatapoint"
+    },
+    {
+        "args": {
+            "address": 40012,
+            "count": 2,
+            "feedbackTopic": null,
+            "label": "Counter2",
+            "publishTopic": "Pub/Counter2",
+            "scanRate": 60.0,
+            "subscribeTopic": null,
+            "unit": 4
+        },
+        "type": "HoldingRegisterDatapoint"
+    },
+    {
+        "args": {
+            "address": 40014,
+            "count": 2,
+            "feedbackTopic": null,
+            "label": "Counter3",
+            "publishTopic": "Pub/Counter3",
+            "scanRate": 60.0,
+            "subscribeTopic": null,
+            "unit": 4
+        },
+        "type": "HoldingRegisterDatapoint"
+    },
+    {
+        "args": {
+            "address": 40016,
+            "count": 2,
+            "feedbackTopic": null,
+            "label": "Counter4",
+            "publishTopic": "Pub/Counter4",
+            "scanRate": 60.0,
+            "subscribeTopic": null,
+            "unit": 4
+        },
+        "type": "HoldingRegisterDatapoint"
+    },
+    {
+        "args": {
+            "address": 40001,
+            "count": 1,
+            "feedbackTopic": null,
+            "label": "Version",
+            "publishTopic": "Pub/Version",
+            "scanRate": 1.0,
+            "subscribeTopic": null,
+            "unit": 4
+        },
+        "type": "HoldingRegisterDatapoint"
+    },
+    {
+        "args": {
+            "address": 40001,
+            "count": 1,
+            "feedbackTopic": null,
+            "label": "Version",
+            "publishTopic": "Pub/Version",
+            "scanRate": 10.0,
+            "subscribeTopic": null,
+            "unit": 4
+        },
+        "type": "HoldingRegisterDatapoint"
+    },
+    {
+        "args": {
+            "address": 40001,
+            "count": 1,
+            "feedbackTopic": "FB/Version",
+            "label": "Version_W",
+            "publishTopic": null,
+            "scanRate": 0.0,
+            "subscribeTopic": "Sub/Version",
+            "unit": 4
+        },
+        "type": "HoldingRegisterDatapoint"
+    }
+]

src/updateRegisterFile.py

@@ -1,24 +1,24 @@
 import datetime
 import RegisterDatapoint
 import pickle
-
+import json
 
 
 with open('registers.pkl', 'rb') as f:
     datapoints = pickle.load(f)
 
-RegisterDatapoint.checkRegisterList(datapoints, reset=True)
-
 newDatapoints = []
 for dp in datapoints:
     ndp = type(dp)()
     for k,v in dp.__dict__.items():
-        ndp.__dict__[k] = v
+        if k != 'logger':
+            ndp.__dict__[k] = v
     newDatapoints.append(ndp)
 
-RegisterDatapoint.checkRegisterList(newDatapoints, reset=True)
-
-with open('registers.pkl', 'wb') as f:
-    pickle.dump(newDatapoints, f)
 
 
+js = json.dumps(newDatapoints, cls=RegisterDatapoint.JsonifyEncoder, sort_keys=True, indent=4)
+print(js)
+    
+
+RegisterDatapoint.saveRegisterList(newDatapoints, 'registers.json')