fix package name, start database handling

This commit is contained in:
Wolfgang Hottgenroth
2017-11-21 17:53:11 +01:00
parent 7b36caf330
commit 5896212ed9
57 changed files with 367 additions and 225 deletions

View File

@ -0,0 +1,53 @@
package de.hottis.measurementCollector;
import java.io.Serializable;
import java.time.LocalDateTime;
import java.util.Map;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
public abstract class ADataObject implements Serializable {
private static final long serialVersionUID = 1L;
final protected Logger logger = LogManager.getRootLogger();
private LocalDateTime timestamp;
private String name;
private Map<String, Object> values;
public ADataObject(LocalDateTime timestamp, String name) {
this.timestamp = timestamp;
this.name = name;
}
public void setValues(Map<String, Object> values) {
this.values = values;
}
abstract public String getTableName();
public LocalDateTime getTimestamp() {
return timestamp;
}
public Map<String, Object> getValues() {
return values;
}
public String getName() {
return name;
}
public String toString() {
StringBuffer sb = new StringBuffer();
sb.append("<");
sb.append(name);
sb.append(", ");
sb.append(timestamp);
sb.append(", ");
sb.append(values.toString());
sb.append(">");
return sb.toString();
}
}

View File

@ -0,0 +1,20 @@
package de.hottis.measurementCollector;
import java.time.LocalDateTime;
import java.util.List;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.openmuc.jmbus.DataRecord;
public abstract class ADataParser {
final protected Logger logger = LogManager.getRootLogger();
protected String name;
public ADataParser(String name) {
this.name = name;
}
abstract public List<ADataObject> parse(LocalDateTime timestamp, List<DataRecord> dataRecords);
}

View File

@ -0,0 +1,39 @@
package de.hottis.measurementCollector;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Properties;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
public abstract class AMessageParser {
final protected Logger logger = LogManager.getRootLogger();
private String topic;
private MyQueue<ADataObject> queue;
protected Properties config;
public AMessageParser(String topic, Properties config, MyQueue<ADataObject> queue) {
this.topic = topic;
this.config = config;
this.queue = queue;
}
public void init() throws MeasurementCollectorException {
}
public String getTopic() {
return this.topic;
}
public void enqueue(List<ADataObject> itemList) throws MeasurementCollectorException {
for (ADataObject ado : itemList) {
queue.enqueue(ado);
logger.debug("message enqueued");
}
}
abstract public void execute(LocalDateTime timestamp, String msgPayload);
}

View File

@ -0,0 +1,122 @@
package de.hottis.measurementCollector;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.Properties;
import java.util.Timer;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
public class DatabaseEngine extends Thread implements ITriggerable {
static final String DATABASE_ENGINE_PERIOD_PROP = "db.period";
static final String DATABASE_URL_PROP = "db.url";
final protected Logger logger = LogManager.getRootLogger();
private Properties config;
private MyQueue<ADataObject> queue;
private boolean stop;
private boolean triggerFlag;
private Timer timer;
private int period;
private String dbUrl;
public DatabaseEngine(Properties config, MyQueue<ADataObject> queue) {
super("MeasurementCollector.DatabaseEngine");
this.config = config;
this.queue = queue;
this.stop = false;
this.triggerFlag = false;
this.period = Integer.parseInt(this.config.getProperty(DATABASE_ENGINE_PERIOD_PROP));
this.dbUrl = this.config.getProperty(DATABASE_URL_PROP);
}
public void requestShutdown() {
logger.info("Shutdown of database engine requested");
this.stop = true;
try {
this.join();
logger.info("Database engine is down");
} catch (InterruptedException e) {
logger.error("Waiting for shutdown of database engine interrupted");
}
}
public synchronized void trigger() {
logger.debug("DatabaseEngine triggered");
triggerFlag = true;
notify();
}
public void init() throws MeasurementCollectorException {
timer = new Timer("DatabaseEngineTrigger");
timer.schedule(new TriggerTimer(this), 0, period * 1000);
}
private String createStatementFromDataObject(ADataObject ado) {
return "";
}
@Override
public synchronized void run() {
while (! stop) {
logger.debug("DatabaseEngine is about to wait for (regularly) " + period + "s");
while (! triggerFlag) {
try {
wait();
} catch (InterruptedException e) {
}
}
triggerFlag = false;
logger.debug("DatabaseEngine has received trigger");
Connection dbCon = null;
try {
int itemCnt = 0;
while (true) {
PreparedStatement pstmt = null;
try {
ADataObject ado = queue.dequeue();
if (ado == null) {
logger.warn("DatabaseEngine found no data");
break;
}
String stmtTxt = createStatementFromDataObject(ado);
pstmt = dbCon.prepareStatement(stmtTxt);
for (Object o : ado.getValues().values()) {
}
itemCnt++;
logger.info("DatabaseEngine received (" + itemCnt + ") " + ado);
} catch (SQLException e) {
logger.error("SQLException in inner database engine loop", e);
} finally {
if (pstmt != null) {
try {
pstmt.close();
pstmt = null;
} catch (SQLException e) {
logger.warn("SQLException when closing statement, nothing will be done");
}
}
}
}
} catch (Exception e) {
logger.error("Exception in outer database engine loop", e);
} finally {
if (dbCon ! null) {
try {
dbCon.close();
dbCon = null;
} catch (SQLException e) {
logger.warn("SQLException when closing connection, nothing will be done");
}
}
}
}
logger.info("Database engine is terminating");
}
}

View File

@ -0,0 +1,24 @@
package de.hottis.measurementCollector;
import java.time.LocalDateTime;
import java.util.HashMap;
public class ElectricEnergyDataObject extends ADataObject {
private static final long serialVersionUID = 1L;
static final String ENERGY_KEY = "energy";
static final String POWER_KEY = "power";
static final String TABLE_NAME = "ElectricEnergy";
public ElectricEnergyDataObject(LocalDateTime timestamp, String name, double energy, double power) {
super(timestamp, name);
HashMap<String, Object> values = new HashMap<String, Object>();
values.put(ENERGY_KEY, energy);
values.put(POWER_KEY, power);
setValues(values);
}
@Override
public String getTableName() {
return TABLE_NAME;
}
}

View File

@ -0,0 +1,23 @@
package de.hottis.measurementCollector;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import org.openmuc.jmbus.DataRecord;
public class FinderOnePhasePowerMeter extends ADataParser {
public FinderOnePhasePowerMeter(String name) {
super(name);
}
public List<ADataObject> parse(LocalDateTime timestamp, List<DataRecord> dataRecords) {
ArrayList<ADataObject> list = new ArrayList<ADataObject>();
ElectricEnergyDataObject tdo = new ElectricEnergyDataObject(timestamp, name,
dataRecords.get(0).getScaledDataValue(), dataRecords.get(4).getScaledDataValue());
list.add(tdo);
return list;
}
}

View File

@ -0,0 +1,23 @@
package de.hottis.measurementCollector;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import org.openmuc.jmbus.DataRecord;
public class FinderThreePhasePowerMeter extends ADataParser {
public FinderThreePhasePowerMeter(String name) {
super(name);
}
public List<ADataObject> parse(LocalDateTime timestamp, List<DataRecord> dataRecords) {
ArrayList<ADataObject> list = new ArrayList<ADataObject>();
ElectricEnergyDataObject tdo = new ElectricEnergyDataObject(timestamp, name,
dataRecords.get(0).getScaledDataValue(), dataRecords.get(17).getScaledDataValue());
list.add(tdo);
return list;
}
}

View File

@ -0,0 +1,25 @@
package de.hottis.measurementCollector;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import org.openmuc.jmbus.DataRecord;
public class HottisFourChannelThermometer extends ADataParser {
public HottisFourChannelThermometer(String name) {
super(name);
}
public List<ADataObject> parse(LocalDateTime timestamp, List<DataRecord> dataRecords) {
ArrayList<ADataObject> list = new ArrayList<ADataObject>();
TemperatureDataObject tdo = new TemperatureDataObject(timestamp, "Hedge", dataRecords.get(5).getScaledDataValue());
list.add(tdo);
tdo = new TemperatureDataObject(timestamp, "House", dataRecords.get(7).getScaledDataValue());
list.add(tdo);
return list;
}
}

View File

@ -0,0 +1,5 @@
package de.hottis.measurementCollector;
public interface ITriggerable {
public void trigger();
}

View File

@ -0,0 +1,124 @@
package de.hottis.measurementCollector;
import java.io.Serializable;
import java.util.Properties;
import javax.jms.Connection;
import javax.jms.DeliveryMode;
import javax.jms.JMSException;
import javax.jms.MessageConsumer;
import javax.jms.MessageProducer;
import javax.jms.ObjectMessage;
import javax.jms.Session;
import javax.jms.Topic;
import org.apache.activemq.ActiveMQConnectionFactory;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
public class JmsTopic<T extends Serializable> implements MyQueue<T> {
static final String JMS_PARSED_DATA_TOPIC = "jms.parseddata.topic";
static final String JMS_CLIENTID_PROP = "jms.clientid";
static final String JMS_BROKER ="jms.broker";
static enum Mode { CONSUMER, PRODUCER, CONSUMER_PRODUCER };
final protected Logger logger = LogManager.getRootLogger();
private String clientId;
private Properties config;
private Connection connection;
private Session session;
private Topic topic;
private MessageProducer producer;
private MessageConsumer consumer;
private String topicTxt;
private String broker;
private Mode mode;
public JmsTopic(Properties config, Mode mode) {
this.config = config;
clientId = this.config.getProperty(JMS_CLIENTID_PROP);
topicTxt = this.config.getProperty(JMS_PARSED_DATA_TOPIC);
broker = this.config.getProperty(JMS_BROKER);
this.mode = mode;
}
public void init() throws MeasurementCollectorException {
try {
ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory(broker);
connectionFactory.setTrustAllPackages(true);
logger.debug("connectionFactory: " + connectionFactory);
connection = connectionFactory.createConnection();
connection.setClientID(clientId);
connection.start();
logger.debug("connection: " + connection);
session = connection.createSession(false, Session.AUTO_ACKNOWLEDGE);
logger.debug("session: " + session);
topic = session.createTopic(topicTxt);
logger.debug("topic: " + topic);
if ((mode == Mode.PRODUCER) || (mode == Mode.CONSUMER_PRODUCER)) {
producer = session.createProducer(topic);
producer.setDeliveryMode(DeliveryMode.NON_PERSISTENT);
logger.debug("producer: " + producer);
} else {
producer = null;
}
if ((mode == Mode.CONSUMER) || (mode == Mode.CONSUMER_PRODUCER)) {
consumer = session.createDurableSubscriber(topic, "DatabaseEngine");
logger.debug("consumer: " + consumer);
} else {
consumer = null;
}
} catch (JMSException e) {
logger.error("JMSException in AJmsQueue.init", e);
throw new MeasurementCollectorException("JMSException in AJmsTopic.init", e);
}
}
public T dequeue() throws MeasurementCollectorException {
if (consumer == null) {
throw new MeasurementCollectorException("This is no consumer");
}
try {
ObjectMessage message = (ObjectMessage) consumer.receiveNoWait();
T item;
if (message != null) {
@SuppressWarnings("unchecked")
T t = (T) message.getObject();
item = t;
logger.debug("message dequeued");
} else {
item = null;
}
return item;
} catch (JMSException e) {
logger.error("JMSException in JmsTopic.dequeue", e);
logger.error("Calling init");
init();
throw new MeasurementCollectorException("JMSException in JmsTopic.dequeue", e);
}
}
public void enqueue(T item) throws MeasurementCollectorException {
if (producer == null) {
throw new MeasurementCollectorException("This is no producer");
}
try {
ObjectMessage message = session.createObjectMessage();
message.setObject(item);
producer.send(message);
logger.debug("message enqueued");
} catch (JMSException e) {
logger.error("JMSException in JmsTopic.enqueue", e);
logger.error("Calling init");
init();
throw new MeasurementCollectorException("JMSException in JmsTopic.enqueue", e);
}
}
}

View File

@ -0,0 +1,98 @@
package de.hottis.measurementCollector;
import java.lang.reflect.Constructor;
import java.time.LocalDateTime;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.openmuc.jmbus.DataRecord;
import org.openmuc.jmbus.MBusMessage;
import org.openmuc.jmbus.VariableDataStructure;
import com.json.parsers.JSONParser;
import com.json.parsers.JsonParserFactory;
public class MBusParser extends AMessageParser {
static final String DATA_PARSER_PROP = "mbus.dataparser";
static final String TOPIC = "IoT/Measurement/MeterbusHub";
private final JSONParser jsonParser;
private HashMap<String, ADataParser> dataParsers;
public MBusParser(Properties config, MyQueue<ADataObject> queue) {
super(TOPIC, config, queue);
JsonParserFactory jsonParserFactory = JsonParserFactory.getInstance();
jsonParser = jsonParserFactory.newJsonParser();
dataParsers = new HashMap<String, ADataParser>();
}
public void registerConfiguredDataParsers() throws MeasurementCollectorException {
try {
@SuppressWarnings("unchecked")
Enumeration<String> propNames = (Enumeration<String>) config.propertyNames();
while (propNames.hasMoreElements()) {
String propName = propNames.nextElement();
if (propName.startsWith(DATA_PARSER_PROP)) {
String[] parserConfigElements = config.get(propName).toString().split(",");
String nameInMsg = parserConfigElements[0];
String nameInDatabase = parserConfigElements[1];
String dataParserClassName = parserConfigElements[2];
Class<?> klass = Class.forName(dataParserClassName);
Constructor<?> constructor = klass.getConstructor(String.class);
ADataParser dataParser = (ADataParser) constructor.newInstance(nameInDatabase);
dataParsers.put(nameInMsg, dataParser);
logger.info(dataParserClassName + " registered for " + nameInMsg + ", " + nameInDatabase);
}
}
}
catch (Exception e) {
throw new MeasurementCollectorException("Exception when registering configured data parsers", e);
}
}
@Override
public void execute(LocalDateTime timestamp, String msgPayload) {
try {
@SuppressWarnings("rawtypes")
Map payloadMap = jsonParser.parseJson(msgPayload);
@SuppressWarnings("rawtypes")
String name = (String)(((Map)(payloadMap.get("metadata"))).get("name"));
@SuppressWarnings("rawtypes")
String mbusMsgTxt = (String)(((Map)(payloadMap.get("data"))).get("telegram"));
String [] octetsTxt = mbusMsgTxt.split(" ");
byte [] octets = new byte[octetsTxt.length];
for (int i = 0; i < octetsTxt.length; i++) {
octets[i] = (byte)(Integer.parseInt(octetsTxt[i], 16) & 0xff);
}
MBusMessage mbusMsg = MBusMessage.decode(octets, octets.length);
VariableDataStructure variableDataStructure = mbusMsg.getVariableDataResponse();
variableDataStructure.decode();
List<DataRecord> dataRecords = variableDataStructure.getDataRecords();
if (dataParsers.containsKey(name)) {
List<ADataObject> measurementItems = dataParsers.get(name).parse(timestamp, dataRecords);
//for (ADataObject ado : measurementItems) {
// logger.debug(ado);
//}
enqueue(measurementItems);
} else {
logger.warn("unknown name: " + name);
}
} catch (Exception e) {
logger.error("Exception when handling mbus message: ", e);
}
}
}

View File

@ -0,0 +1,35 @@
package de.hottis.measurementCollector;
import java.util.Properties;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
public class MeasurementCollector {
static final String PROPS_FILENAME = "measurementCollector.props";
static final Logger logger = LogManager.getRootLogger();
public static void main(String[] args) throws Exception {
logger.info("MeasurementCollector starting");
final Properties config = new Properties();
config.load(MeasurementCollector.class.getClassLoader().getResourceAsStream(PROPS_FILENAME));
logger.debug("Configuration loaded");
MqttReceiver mqttReceiver = new MqttReceiver(config);
mqttReceiver.connect();
logger.debug("MqttReceiver started");
JmsTopic<ADataObject> queue = new JmsTopic<ADataObject>(config, JmsTopic.Mode.PRODUCER);
queue.init();
MBusParser mbusParser = new MBusParser(config, queue);
mbusParser.init();
mbusParser.registerConfiguredDataParsers();
mqttReceiver.registerParser(mbusParser);
logger.debug("MBusParser started");
}
}

View File

@ -0,0 +1,13 @@
package de.hottis.measurementCollector;
@SuppressWarnings("serial")
public class MeasurementCollectorException extends Exception {
public MeasurementCollectorException(String msg, Throwable cause) {
super(msg, cause);
}
public MeasurementCollectorException(String msg) {
super(msg);
}
}

View File

@ -0,0 +1,30 @@
package de.hottis.measurementCollector;
import java.util.Properties;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
public class MeasurementDatabaseEngine {
static final String PROPS_FILENAME = "measurementDataEngine.props";
static final Logger logger = LogManager.getRootLogger();
public static void main(String[] args) throws Exception {
logger.info("MeasurementDatabaseEngine starting");
final Properties config = new Properties();
config.load(MeasurementDatabaseEngine.class.getClassLoader().getResourceAsStream(PROPS_FILENAME));
logger.debug("Configuration loaded");
JmsTopic<ADataObject> queue = new JmsTopic<ADataObject>(config, JmsTopic.Mode.CONSUMER);
queue.init();
DatabaseEngine databaseEngine = new DatabaseEngine(config, queue);
databaseEngine.init();
databaseEngine.start();
logger.debug("DatabaseEngine started");
}
}

View File

@ -0,0 +1,112 @@
package de.hottis.measurementCollector;
import java.time.LocalDateTime;
import java.util.HashMap;
import java.util.Properties;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken;
import org.eclipse.paho.client.mqttv3.MqttCallbackExtended;
import org.eclipse.paho.client.mqttv3.MqttClient;
import org.eclipse.paho.client.mqttv3.MqttConnectOptions;
import org.eclipse.paho.client.mqttv3.MqttException;
import org.eclipse.paho.client.mqttv3.MqttMessage;
public class MqttReceiver {
final protected Logger logger = LogManager.getRootLogger();
static final String MQTT_BROKER_PROP = "mqtt.broker";
static final String MQTT_CLIENTID_PROP = "mqtt.clientid";
static final String MQTT_USERNAME_PROP = "mqtt.username";
static final String MQTT_PASSWORD_PROP = "mqtt.password";
class Callback implements MqttCallbackExtended {
public void messageArrived(String topic, MqttMessage payload) {
parsers.get(topic).execute(LocalDateTime.now(), payload.toString());
}
public void connectComplete(boolean reconnect, java.lang.String serverURI) {
logger.info("Connection established for " + serverURI);
}
public void connectionLost(java.lang.Throwable cause) {
logger.error("Connection lost, cause: " + cause.toString());
reconnect();
}
public void deliveryComplete(IMqttDeliveryToken token) {
}
}
private final String broker;
private final String clientId;
private final MqttConnectOptions connOpts;
private MqttClient client;
private HashMap<String, AMessageParser> parsers;
private final Callback callback = new Callback();
public MqttReceiver(Properties config) {
broker = config.getProperty(MQTT_BROKER_PROP, "localhost");
clientId = config.getProperty(MQTT_CLIENTID_PROP, "MeasurementCollector.MqttReceiver");
connOpts = new MqttConnectOptions();
String username = config.getProperty(MQTT_USERNAME_PROP);
String password = config.getProperty(MQTT_PASSWORD_PROP);
if (username != null && password != null) {
connOpts.setUserName(username);
connOpts.setPassword(password.toCharArray());
}
parsers = new HashMap<String, AMessageParser>();
}
public void connect() throws MeasurementCollectorException {
try {
client = new MqttClient(broker, clientId);
client.setCallback(callback);
client.connect(connOpts);
logger.info("Connected");
} catch (MqttException e) {
throw new MeasurementCollectorException("MqttReceiver.connect", e);
}
}
public void reconnect() {
logger.error("reconnect called");
if (! client.isConnected()) {
while (true) {
try {
client.connect(connOpts);
for (String topic : parsers.keySet()) {
client.subscribe(topic);
logger.info("Re-Subscribed: " + topic);
}
logger.error("reconnecting successfully completed");
break;
} catch (MqttException e) {
logger.error("Exception during reconnection: " + e.toString());
try {
Thread.sleep(10*1000);
} catch (InterruptedException e1) {
}
}
}
} else {
logger.error("client is still connected");
}
}
public void registerParser(AMessageParser parser) throws MeasurementCollectorException {
try {
parsers.put(parser.getTopic(), parser);
client.subscribe(parser.getTopic());
logger.info("Subscribed: " + parser.getTopic());
} catch (MqttException e) {
throw new MeasurementCollectorException("MqttReceiver.registerParser", e);
}
}
}

View File

@ -0,0 +1,8 @@
package de.hottis.measurementCollector;
import java.io.Serializable;
public interface MyQueue<T extends Serializable> {
public T dequeue() throws MeasurementCollectorException;
public void enqueue(T item) throws MeasurementCollectorException;
}

View File

@ -0,0 +1,21 @@
package de.hottis.measurementCollector;
import java.time.LocalDateTime;
import java.util.HashMap;
public class TemperatureDataObject extends ADataObject {
private static final long serialVersionUID = 1L;
static final String TEMPERATURE_KEY = "temperature";
static final String TABLE_NAME = "Temperature";
public TemperatureDataObject(LocalDateTime timestamp, String name, double temperature) {
super(timestamp, name);
HashMap<String, Object> values = new HashMap<String, Object>();
values.put(TEMPERATURE_KEY, temperature);
setValues(values);
}
public String getTableName() {
return TABLE_NAME;
}
}

View File

@ -0,0 +1,15 @@
package de.hottis.measurementCollector;
import java.util.TimerTask;
public class TriggerTimer extends TimerTask {
private ITriggerable triggerable;
public TriggerTimer(ITriggerable triggerable) {
this.triggerable = triggerable;
}
public void run() {
triggerable.trigger();
}
}