From d14b83f3efa0e730f7c5d497fd5d716bc81147b8 Mon Sep 17 00:00:00 2001 From: Wolfgang Hottgenroth Date: Mon, 14 May 2018 14:48:43 +0200 Subject: [PATCH] instancify --- MqttMongoNodejs.conf | 19 +++ package-lock.json | 329 ++++++++++++++++++++++++++----------------- package.json | 4 + src/MqttMongo.ts | 147 +++++++++++++++++++ src/config.ts | 19 +++ src/log.ts | 52 +++++++ src/main.ts | 187 +++--------------------- 7 files changed, 467 insertions(+), 290 deletions(-) create mode 100644 MqttMongoNodejs.conf create mode 100644 src/MqttMongo.ts create mode 100644 src/config.ts create mode 100644 src/log.ts diff --git a/MqttMongoNodejs.conf b/MqttMongoNodejs.conf new file mode 100644 index 0000000..9707a30 --- /dev/null +++ b/MqttMongoNodejs.conf @@ -0,0 +1,19 @@ +{ + "brokerUrl": "mqtt://127.0.0.1:1883", + "brokerUser": "", + "brokerPass": "", + "brokerCa": "", + "mongoDbUrl": "mongodb://localhost/testdb", + "verbose": false, + "instances": [ + { + "instanceId": "testInstance", + "collection": "testcollection", + "topics": [ "topic1", "topic2", "topic3" ], + "encapsulate": false, + "parsePayload": false + } + ] +} + + diff --git a/package-lock.json b/package-lock.json index c46332a..1c647d8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,24 +10,39 @@ "integrity": "sha512-PMa4nkRhLaqwsXQDDTzGbTyCIpej0ERznyAP9fyuGnlsmUbcC4Y25mdqjibYjkOPNyK/BWWUKneruaKHcS3Q8g==", "dev": true, "requires": { - "@types/node": "7.0.61" + "@types/node": "*" } }, + "@types/command-line-args": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.0.0.tgz", + "integrity": "sha512-4eOPXyn5DmP64MCMF8ePDvdlvlzt2a+F8ZaVjqmh2yFCpGjc1kI3kGnCFYX9SCsGTjQcWIyVZ86IHCEyjy/MNg==", + "dev": true + }, "@types/events": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@types/events/-/events-1.2.0.tgz", "integrity": "sha512-KEIlhXnIutzKwRbQkGWb/I4HFqBuUykAdHgDED6xqwXJfONCjF5VoE0cXEiurh3XauygxzeDzgtXUqvLkxFzzA==", "dev": true }, + "@types/moment": { + "version": "2.13.0", + "resolved": "https://registry.npmjs.org/@types/moment/-/moment-2.13.0.tgz", + "integrity": "sha1-YE69GJvDvDShVIaJQE5hoqSqyJY=", + "dev": true, + "requires": { + "moment": "*" + } + }, "@types/mongodb": { "version": "2.2.20", "resolved": "https://registry.npmjs.org/@types/mongodb/-/mongodb-2.2.20.tgz", "integrity": "sha512-EcsDWWppvtZ/C70+oDxYmbMNtckQ4dD4v60vI6HBamWuDNrblUEMvI6LDicpbE6Mpt6nhq/T7qzUOsy9cRgvYw==", "dev": true, "requires": { - "@types/bson": "1.0.8", - "@types/events": "1.2.0", - "@types/node": "7.0.61" + "@types/bson": "*", + "@types/events": "*", + "@types/node": "*" } }, "@types/mqtt": { @@ -36,7 +51,7 @@ "integrity": "sha1-eGV5AADMijEiQurZoOIJuiqmhtQ=", "dev": true, "requires": { - "@types/node": "7.0.61" + "@types/node": "*" } }, "@types/node": { @@ -55,6 +70,23 @@ "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=" }, + "argv-tools": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/argv-tools/-/argv-tools-0.1.1.tgz", + "integrity": "sha512-Cc0dBvx4dvrjjKpyDA6w8RlNAw8Su30NvZbWl/Tv9ZALEVlLVkWQiHMi84Q0xNfpVuSaiQbYkdmWK8g1PLGhKw==", + "requires": { + "array-back": "^2.0.0", + "find-replace": "^2.0.1" + } + }, + "array-back": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-2.0.0.tgz", + "integrity": "sha512-eJv4pLLufP3g5kcZry0j6WXpIbzYw9GUB4mVJZno9wfwiBxbizTnHCw3VJb07cBihbFX48Y7oSrW9y+gt4glyw==", + "requires": { + "typical": "^2.6.1" + } + }, "async-limiter": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz", @@ -70,8 +102,8 @@ "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz", "integrity": "sha512-e8tQYnZodmebYDWGH7KMRvtzKXaJHx3BbilrgZCfvyLUYdKpK1t5PSPmpkny/SgiTSCnjfLW7v5rlONXVFkQEA==", "requires": { - "readable-stream": "2.3.6", - "safe-buffer": "5.1.2" + "readable-stream": "^2.3.5", + "safe-buffer": "^5.1.1" } }, "brace-expansion": { @@ -79,7 +111,7 @@ "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", "requires": { - "balanced-match": "1.0.0", + "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, @@ -103,8 +135,8 @@ "resolved": "https://registry.npmjs.org/callback-stream/-/callback-stream-1.1.0.tgz", "integrity": "sha1-RwGlEmbwbgbqpx/BcjOCLYdfSQg=", "requires": { - "inherits": "2.0.3", - "readable-stream": "2.3.6" + "inherits": "^2.0.1", + "readable-stream": "> 1.0.0 < 3.0.0" } }, "chalk": { @@ -112,11 +144,11 @@ "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", "requires": { - "ansi-styles": "2.2.1", - "escape-string-regexp": "1.0.5", - "has-ansi": "2.0.0", - "strip-ansi": "3.0.1", - "supports-color": "2.0.0" + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" } }, "chalk-console": { @@ -124,9 +156,21 @@ "resolved": "https://registry.npmjs.org/chalk-console/-/chalk-console-1.0.1.tgz", "integrity": "sha1-os8GLn+P+6TsAUzQqgcZfYVVgrw=", "requires": { - "chalk": "1.1.3", - "date-utils": "1.2.21", - "lodash": "3.10.1" + "chalk": "^1.0.0", + "date-utils": "^1.2.16", + "lodash": "^3.7.0" + } + }, + "command-line-args": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-5.0.2.tgz", + "integrity": "sha512-/qPcbL8zpqg53x4rAaqMFlRV4opN3pbla7I7k9x8kyOBMQoGT6WltjN6sXZuxOXw6DgdK7Ad+ijYS5gjcr7vlA==", + "requires": { + "argv-tools": "^0.1.1", + "array-back": "^2.0.0", + "find-replace": "^2.0.1", + "lodash.camelcase": "^4.3.0", + "typical": "^2.6.1" } }, "commander": { @@ -139,8 +183,8 @@ "resolved": "https://registry.npmjs.org/commist/-/commist-1.0.0.tgz", "integrity": "sha1-wMNSUBz29S6RJOPvicmAbiAi6+8=", "requires": { - "leven": "1.0.2", - "minimist": "1.2.0" + "leven": "^1.0.0", + "minimist": "^1.1.0" } }, "concat-map": { @@ -153,10 +197,10 @@ "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", "requires": { - "buffer-from": "1.0.0", - "inherits": "2.0.3", - "readable-stream": "2.3.6", - "typedarray": "0.0.6" + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^2.2.2", + "typedarray": "^0.0.6" } }, "core-util-is": { @@ -174,10 +218,10 @@ "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.5.4.tgz", "integrity": "sha512-JzYSLYMhoVVBe8+mbHQ4KgpvHpm0DZpJuL8PY93Vyv1fW7jYJ90LoXa1di/CVbJM+TgMs91rbDapE/RNIfnJsA==", "requires": { - "end-of-stream": "1.4.1", - "inherits": "2.0.3", - "readable-stream": "2.3.6", - "stream-shift": "1.0.0" + "end-of-stream": "^1.0.0", + "inherits": "^2.0.1", + "readable-stream": "^2.0.0", + "stream-shift": "^1.0.0" } }, "end-of-stream": { @@ -185,7 +229,7 @@ "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz", "integrity": "sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q==", "requires": { - "once": "1.4.0" + "once": "^1.4.0" } }, "es6-promise": { @@ -203,6 +247,15 @@ "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.1.tgz", "integrity": "sha1-p1Xqe8Gt/MWjHOfnYtuq3F5jZEQ=" }, + "find-replace": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/find-replace/-/find-replace-2.0.1.tgz", + "integrity": "sha512-LzDo3Fpa30FLIBsh6DCDnMN1KW2g4QKkqKmejlImgWY67dDFPX/x9Kh/op/GK522DchQXEvDi/wD48HKW49XOQ==", + "requires": { + "array-back": "^2.0.0", + "test-value": "^3.0.0" + } + }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -213,12 +266,12 @@ "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==", "requires": { - "fs.realpath": "1.0.0", - "inflight": "1.0.6", - "inherits": "2.0.3", - "minimatch": "3.0.4", - "once": "1.4.0", - "path-is-absolute": "1.0.1" + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" } }, "glob-parent": { @@ -226,8 +279,8 @@ "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-3.1.0.tgz", "integrity": "sha1-nmr2KZ2NO9K9QEMIMr0RPfkGxa4=", "requires": { - "is-glob": "3.1.0", - "path-dirname": "1.0.2" + "is-glob": "^3.1.0", + "path-dirname": "^1.0.0" } }, "glob-stream": { @@ -235,16 +288,16 @@ "resolved": "https://registry.npmjs.org/glob-stream/-/glob-stream-6.1.0.tgz", "integrity": "sha1-cEXJlBOz65SIjYOrRtC0BMx73eQ=", "requires": { - "extend": "3.0.1", - "glob": "7.1.2", - "glob-parent": "3.1.0", - "is-negated-glob": "1.0.0", - "ordered-read-streams": "1.0.1", - "pumpify": "1.4.0", - "readable-stream": "2.3.6", - "remove-trailing-separator": "1.1.0", - "to-absolute-glob": "2.0.2", - "unique-stream": "2.2.1" + "extend": "^3.0.0", + "glob": "^7.1.1", + "glob-parent": "^3.1.0", + "is-negated-glob": "^1.0.0", + "ordered-read-streams": "^1.0.0", + "pumpify": "^1.3.5", + "readable-stream": "^2.1.5", + "remove-trailing-separator": "^1.0.1", + "to-absolute-glob": "^2.0.0", + "unique-stream": "^2.0.2" } }, "has-ansi": { @@ -252,7 +305,7 @@ "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=", "requires": { - "ansi-regex": "2.1.1" + "ansi-regex": "^2.0.0" } }, "help-me": { @@ -260,10 +313,10 @@ "resolved": "https://registry.npmjs.org/help-me/-/help-me-1.1.0.tgz", "integrity": "sha1-jy1QjQYAtKRW2i8IZVbn5cBWo8Y=", "requires": { - "callback-stream": "1.1.0", - "glob-stream": "6.1.0", - "through2": "2.0.3", - "xtend": "4.0.1" + "callback-stream": "^1.0.2", + "glob-stream": "^6.1.0", + "through2": "^2.0.1", + "xtend": "^4.0.0" } }, "inflight": { @@ -271,8 +324,8 @@ "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", "requires": { - "once": "1.4.0", - "wrappy": "1.0.2" + "once": "^1.3.0", + "wrappy": "1" } }, "inherits": { @@ -285,8 +338,8 @@ "resolved": "https://registry.npmjs.org/is-absolute/-/is-absolute-1.0.0.tgz", "integrity": "sha512-dOWoqflvcydARa360Gvv18DZ/gRuHKi2NU/wU5X1ZFzdYfH29nkiNZsF3mp4OJ3H4yo9Mx8A/uAGNzpzPN3yBA==", "requires": { - "is-relative": "1.0.0", - "is-windows": "1.0.2" + "is-relative": "^1.0.0", + "is-windows": "^1.0.1" } }, "is-extglob": { @@ -299,7 +352,7 @@ "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-3.1.0.tgz", "integrity": "sha1-e6WuJCF4BKxwcHuWkiVnSGzD6Eo=", "requires": { - "is-extglob": "2.1.1" + "is-extglob": "^2.1.0" } }, "is-negated-glob": { @@ -312,7 +365,7 @@ "resolved": "https://registry.npmjs.org/is-relative/-/is-relative-1.0.0.tgz", "integrity": "sha512-Kw/ReK0iqwKeu0MITLFuj0jbPAmEiOsIwyIXvvbfa6QfmN9pkD1M+8pdk7Rl/dTKbH34/XBFMbgD4iMJhLQbGA==", "requires": { - "is-unc-path": "1.0.0" + "is-unc-path": "^1.0.0" } }, "is-unc-path": { @@ -320,7 +373,7 @@ "resolved": "https://registry.npmjs.org/is-unc-path/-/is-unc-path-1.0.0.tgz", "integrity": "sha512-mrGpVd0fs7WWLfVsStvgF6iEJnbjDFZh9/emhRDcGWTduTfNHd9CHeUwH3gYIjdbwo4On6hunkztwOaAw0yllQ==", "requires": { - "unc-path-regex": "0.1.2" + "unc-path-regex": "^0.1.2" } }, "is-windows": { @@ -338,7 +391,7 @@ "resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.0.1.tgz", "integrity": "sha1-mnWdOcXy/1A/1TAGRu1EX4jE+a8=", "requires": { - "jsonify": "0.0.0" + "jsonify": "~0.0.0" } }, "jsonify": { @@ -356,12 +409,17 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-3.10.1.tgz", "integrity": "sha1-W/Rejkm6QYnhfUgnid/RW9FAt7Y=" }, + "lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha1-soqmKIorn8ZRA1x3EfZathkDMaY=" + }, "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", "requires": { - "brace-expansion": "1.1.11" + "brace-expansion": "^1.1.7" } }, "minimist": { @@ -369,6 +427,11 @@ "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" }, + "moment": { + "version": "2.22.1", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.22.1.tgz", + "integrity": "sha512-shJkRTSebXvsVqk56I+lkb2latjBs8I+pc2TzWc545y2iFnSjm7Wg0QMh+ZWcdSLQyGEau5jI8ocnmkyTgr9YQ==" + }, "mongodb": { "version": "2.2.35", "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-2.2.35.tgz", @@ -384,13 +447,13 @@ "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.2.7.tgz", "integrity": "sha1-BwV6y+JGeyIELTb5jFrVBwVOlbE=", "requires": { - "buffer-shims": "1.0.0", - "core-util-is": "1.0.2", - "inherits": "2.0.3", - "isarray": "1.0.0", - "process-nextick-args": "1.0.7", - "string_decoder": "1.0.3", - "util-deprecate": "1.0.2" + "buffer-shims": "~1.0.0", + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "~1.0.0", + "process-nextick-args": "~1.0.6", + "string_decoder": "~1.0.0", + "util-deprecate": "~1.0.1" } } } @@ -400,8 +463,8 @@ "resolved": "https://registry.npmjs.org/mongodb-core/-/mongodb-core-2.1.19.tgz", "integrity": "sha512-Jt4AtWUkpuW03kRdYGxga4O65O1UHlFfvvInslEfLlGi+zDMxbBe3J2NVmN9qPJ957Mn6Iz0UpMtV80cmxCVxw==", "requires": { - "bson": "1.0.6", - "require_optional": "1.0.1" + "bson": "~1.0.4", + "require_optional": "~1.0.0" } }, "mqtt": { @@ -409,19 +472,19 @@ "resolved": "https://registry.npmjs.org/mqtt/-/mqtt-2.17.0.tgz", "integrity": "sha512-eYeK5G/GQcdP/AOrGQMUULX7QvBXt3I9bfmgNkzMTsdSR1ywJQhK1iCYPrhh+rtRl3eUSJwEbO+oZx/Q51uHaw==", "requires": { - "commist": "1.0.0", - "concat-stream": "1.6.2", - "end-of-stream": "1.4.1", - "help-me": "1.1.0", - "inherits": "2.0.3", - "minimist": "1.2.0", - "mqtt-packet": "5.5.0", - "pump": "3.0.0", - "readable-stream": "2.3.6", - "reinterval": "1.1.0", - "split2": "2.2.0", - "websocket-stream": "5.1.2", - "xtend": "4.0.1" + "commist": "^1.0.0", + "concat-stream": "^1.6.2", + "end-of-stream": "^1.4.1", + "help-me": "^1.0.1", + "inherits": "^2.0.3", + "minimist": "^1.2.0", + "mqtt-packet": "^5.5.0", + "pump": "^3.0.0", + "readable-stream": "^2.3.5", + "reinterval": "^1.1.0", + "split2": "^2.1.1", + "websocket-stream": "^5.1.2", + "xtend": "^4.0.1" } }, "mqtt-packet": { @@ -429,10 +492,10 @@ "resolved": "https://registry.npmjs.org/mqtt-packet/-/mqtt-packet-5.5.0.tgz", "integrity": "sha512-kR+Uw+r9rjUFSLZutmaAhjL4Y1asKLMTwE++PP0iuApJuc+zItE5v2LluQN2K3Pri5e2+K4V++QDjqGTgle/+A==", "requires": { - "bl": "1.2.2", - "inherits": "2.0.3", - "process-nextick-args": "2.0.0", - "safe-buffer": "5.1.2" + "bl": "^1.2.1", + "inherits": "^2.0.3", + "process-nextick-args": "^2.0.0", + "safe-buffer": "^5.1.0" }, "dependencies": { "process-nextick-args": { @@ -447,7 +510,7 @@ "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", "requires": { - "wrappy": "1.0.2" + "wrappy": "1" } }, "ordered-read-streams": { @@ -455,7 +518,7 @@ "resolved": "https://registry.npmjs.org/ordered-read-streams/-/ordered-read-streams-1.0.1.tgz", "integrity": "sha1-d8DLN8QVJdZBZtmQ/61+xqDhNj4=", "requires": { - "readable-stream": "2.3.6" + "readable-stream": "^2.0.1" } }, "path-dirname": { @@ -478,8 +541,8 @@ "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", "requires": { - "end-of-stream": "1.4.1", - "once": "1.4.0" + "end-of-stream": "^1.1.0", + "once": "^1.3.1" } }, "pumpify": { @@ -487,9 +550,9 @@ "resolved": "https://registry.npmjs.org/pumpify/-/pumpify-1.4.0.tgz", "integrity": "sha512-2kmNR9ry+Pf45opRVirpNuIFotsxUGLaYqxIwuR77AYrYRMuFCz9eryHBS52L360O+NcR383CL4QYlMKPq4zYA==", "requires": { - "duplexify": "3.5.4", - "inherits": "2.0.3", - "pump": "2.0.1" + "duplexify": "^3.5.3", + "inherits": "^2.0.3", + "pump": "^2.0.0" }, "dependencies": { "pump": { @@ -497,8 +560,8 @@ "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz", "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==", "requires": { - "end-of-stream": "1.4.1", - "once": "1.4.0" + "end-of-stream": "^1.1.0", + "once": "^1.3.1" } } } @@ -513,13 +576,13 @@ "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "requires": { - "core-util-is": "1.0.2", - "inherits": "2.0.3", - "isarray": "1.0.0", - "process-nextick-args": "2.0.0", - "safe-buffer": "5.1.2", - "string_decoder": "1.1.1", - "util-deprecate": "1.0.2" + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" }, "dependencies": { "process-nextick-args": { @@ -532,7 +595,7 @@ "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "requires": { - "safe-buffer": "5.1.2" + "safe-buffer": "~5.1.0" } } } @@ -552,8 +615,8 @@ "resolved": "https://registry.npmjs.org/require_optional/-/require_optional-1.0.1.tgz", "integrity": "sha512-qhM/y57enGWHAe3v/NcwML6a3/vfESLe/sGM2dII+gEO0BpKRUkWZow/tyloNqJyN6kXSl3RyyM8Ll5D/sJP8g==", "requires": { - "resolve-from": "2.0.0", - "semver": "5.5.0" + "resolve-from": "^2.0.0", + "semver": "^5.1.0" } }, "resolve-from": { @@ -576,7 +639,7 @@ "resolved": "https://registry.npmjs.org/split2/-/split2-2.2.0.tgz", "integrity": "sha512-RAb22TG39LhI31MbreBgIuKiIKhVsawfTgEGqKHTK87aG+ul/PB8Sqoi3I7kVdRWiCfrKxK3uo4/YUkpNvhPbw==", "requires": { - "through2": "2.0.3" + "through2": "^2.0.2" } }, "stream-shift": { @@ -589,7 +652,7 @@ "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz", "integrity": "sha512-4AH6Z5fzNNBcH+6XDMfA/BTt87skxqJlO0lAh3Dker5zThcAxG6mKz+iGu308UKoPPQ8Dcqx/4JhujzltRa+hQ==", "requires": { - "safe-buffer": "5.1.2" + "safe-buffer": "~5.1.0" } }, "strip-ansi": { @@ -597,7 +660,7 @@ "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", "requires": { - "ansi-regex": "2.1.1" + "ansi-regex": "^2.0.0" } }, "supports-color": { @@ -605,13 +668,22 @@ "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=" }, + "test-value": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/test-value/-/test-value-3.0.0.tgz", + "integrity": "sha512-sVACdAWcZkSU9x7AOmJo5TqE+GyNJknHaHsMrR6ZnhjVlVN9Yx6FjHrsKZ3BjIpPCT68zYesPWkakrNupwfOTQ==", + "requires": { + "array-back": "^2.0.0", + "typical": "^2.6.1" + } + }, "through2": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.3.tgz", "integrity": "sha1-AARWmzfHx0ujnEPzzteNGtlBQL4=", "requires": { - "readable-stream": "2.3.6", - "xtend": "4.0.1" + "readable-stream": "^2.1.5", + "xtend": "~4.0.1" } }, "through2-filter": { @@ -619,8 +691,8 @@ "resolved": "https://registry.npmjs.org/through2-filter/-/through2-filter-2.0.0.tgz", "integrity": "sha1-YLxVoNrLdghdsfna6Zq0P4PWIuw=", "requires": { - "through2": "2.0.3", - "xtend": "4.0.1" + "through2": "~2.0.0", + "xtend": "~4.0.0" } }, "to-absolute-glob": { @@ -628,8 +700,8 @@ "resolved": "https://registry.npmjs.org/to-absolute-glob/-/to-absolute-glob-2.0.2.tgz", "integrity": "sha1-GGX0PZ50sIItufFFt4z/fQ98hJs=", "requires": { - "is-absolute": "1.0.0", - "is-negated-glob": "1.0.0" + "is-absolute": "^1.0.0", + "is-negated-glob": "^1.0.0" } }, "typedarray": { @@ -643,6 +715,11 @@ "integrity": "sha512-K7g15Bb6Ra4lKf7Iq2l/I5/En+hLIHmxWZGq3D4DIRNFxMNV6j2SHSvDOqs2tGd4UvD/fJvrwopzQXjLrT7Itw==", "dev": true }, + "typical": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/typical/-/typical-2.6.1.tgz", + "integrity": "sha1-XAgOXWYcu+OCWdLnCjxyU+hziB0=" + }, "ultron": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.1.tgz", @@ -658,8 +735,8 @@ "resolved": "https://registry.npmjs.org/unique-stream/-/unique-stream-2.2.1.tgz", "integrity": "sha1-WqADz76Uxf+GbE59ZouxxNuts2k=", "requires": { - "json-stable-stringify": "1.0.1", - "through2-filter": "2.0.0" + "json-stable-stringify": "^1.0.0", + "through2-filter": "^2.0.0" } }, "util-deprecate": { @@ -672,12 +749,12 @@ "resolved": "https://registry.npmjs.org/websocket-stream/-/websocket-stream-5.1.2.tgz", "integrity": "sha512-lchLOk435iDWs0jNuL+hiU14i3ERSrMA0IKSiJh7z6X/i4XNsutBZrtqu2CPOZuA4G/zabiqVAos0vW+S7GEVw==", "requires": { - "duplexify": "3.5.4", - "inherits": "2.0.3", - "readable-stream": "2.3.6", - "safe-buffer": "5.1.2", - "ws": "3.3.3", - "xtend": "4.0.1" + "duplexify": "^3.5.1", + "inherits": "^2.0.1", + "readable-stream": "^2.3.3", + "safe-buffer": "^5.1.1", + "ws": "^3.2.0", + "xtend": "^4.0.0" } }, "wrappy": { @@ -690,9 +767,9 @@ "resolved": "https://registry.npmjs.org/ws/-/ws-3.3.3.tgz", "integrity": "sha512-nnWLa/NwZSt4KQJu51MYlCcSQ5g7INpOrOMt4XV8j4dqTXdmlUmSHQ8/oLC069ckre0fRsgfvsKwbTdtKLCDkA==", "requires": { - "async-limiter": "1.0.0", - "safe-buffer": "5.1.2", - "ultron": "1.1.1" + "async-limiter": "~1.0.0", + "safe-buffer": "~5.1.0", + "ultron": "~1.1.0" } }, "xtend": { diff --git a/package.json b/package.json index 70b7ee2..c19fdc2 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,8 @@ "url": "git@gitlab.com:wolutator/MqttMongoNodejs.git" }, "devDependencies": { + "@types/command-line-args": "^5.0.0", + "@types/moment": "^2.13.0", "@types/mongodb": "^2.2.0", "@types/mqtt": "0.0.34", "@types/node": "^7.0.14", @@ -22,7 +24,9 @@ }, "dependencies": { "chalk-console": "^1.0.1", + "command-line-args": "^5.0.2", "commander": "^2.15.1", + "moment": "^2.22.1", "mongodb": "^2.2.26", "mqtt": "^2.6.2", "queuejs": "^0.1.0", diff --git a/src/MqttMongo.ts b/src/MqttMongo.ts new file mode 100644 index 0000000..a3e476e --- /dev/null +++ b/src/MqttMongo.ts @@ -0,0 +1,147 @@ +import * as Mqtt from 'mqtt' +import * as Mongo from 'mongodb' +import * as Events from 'events' +import * as Queue from './queue' +import * as MqttMessage from './mqtt_message' +import * as logger from './log' + + +export default class MqttMongo extends Events.EventEmitter { + + private options : any + private msgCnt : number = 0 + private startTime: Date + + constructor(options : any) { + super() + this.options = options + this.startTime = new Date() + this.on('reconnectDatabase', this.connectToDatabase) + } + + private dbHandle : Mongo.Db; + private dbReady : boolean = false; + private queue : Queue.Queue = new Queue.Queue() + private mqttClient : Mqtt.Client + private heartbeatTimer : NodeJS.Timer + + logerror(msg: string) { + logger.error(`${this.options.instanceId}: ${msg}`) + } + + logwarn(msg: string) { + logger.warn(`${this.options.instanceId}: ${msg}`) + } + + loginfo(msg: string) { + logger.info(`${this.options.instanceId}: ${msg}`) + } + +connectToDatabase() { + this.loginfo("About to connect to database") + Mongo.MongoClient.connect(this.options.database) + .then( + (tmpDbHandle: Mongo.Db) => { + this.dbHandle = tmpDbHandle + this.dbReady = true; + this.loginfo("Database connected") + this.dbHandle.on('reconnectFailed', (err : any) => { this.logwarn(`Error on database ${err}`) }) + this.dbHandle.on('reconnect', () => { + this.loginfo("Reconnect on database") + this.dbReady = true + this.queue.knock() + }) + this.dbHandle.on('timeout', () => { this.logwarn("Timeout on database") }) + this.dbHandle.on('close', () => { + this.loginfo("Close on database") + this.dbReady = false + }) + }, + (err: String) => { + this.logerror(`Unable to connect to database: ${err}`) + this.dbReady = false + } + ) + } + + connectToBroker() { + this.queue.on('data', () => { + if (this.dbReady || true) { + while (! this.queue.isEmpty()) { + let msg : MqttMessage.MqttMessage = this.queue.deq() + if (this.options.verbose) { + this.loginfo(`Something in the queue: ${JSON.stringify(msg)}`) + } + let coll = this.dbHandle.collection(this.options.collection) + coll.insertOne(msg.getMessage()) + .then( + (r) => { + if (this.options.verbose) { + this.loginfo(`Successfully inserted into database ${JSON.stringify(msg.getMessage())}`) + } + }, + (err) => { + this.logerror(`Error when trying to insert into database ${err}`) + if (! this.dbReady) { + this.loginfo("Error occured while database connection is lost, re-enqueue msg.") + this.queue.reenq(msg) + this.emit('reconnectDatabase') + } else { + this.logerror(`Message ${JSON.stringify(msg.getMessage())} is lost`) + } + } + ) + } + } else { + // this.loginfo("Database currently not available, not reading from stream") + } + }) + + this.mqttClient = Mqtt.connect(this.options.broker) + this.mqttClient.on('offline', () => { console.warn("MQTT client is offline") }) + this.mqttClient.on('reconnect', () => { console.warn("MQTT client is reconnecting") }) + this.mqttClient.on('close', () => { console.warn("MQTT connection closed") }) + + this.mqttClient.on('connect', () => { + this.loginfo("MQTT broker connected") + this.options.topics.forEach((topic: string) => { + this.mqttClient.subscribe(topic) + this.loginfo(`Subscribed to ${topic}`) + }) + this.mqttClient.publish(`MqttMongo/Status/${this.options.instanceId}`, 'hello, started up') + }) + + this.mqttClient.on('message', (topic : string, messageBuf : Buffer) => { + this.msgCnt++; + let message = messageBuf.toString('UTF-8') + if (this.options.verbose) { + this.loginfo(`Message received ${this.msgCnt}, topic ${topic}, payload ${message}`) + } + + try { + this.queue.enq(new MqttMessage.MqttMessage(topic, message, this.options.encapsulate, this.options.parsePayload)) + } catch (e) { + this.logerror(`Error while parsing Mqtt message, topic '${topic}', message '${message}' , ${e.toString()}`) + } + }) + } + + setupHeartbeat() { + this.heartbeatTimer = setInterval(() => { + let uptime : number = (new Date().getTime() - this.startTime.getTime()) / 1000 + let statusMsg = `{'Uptime': ${uptime}, 'MessageCount': ${this.msgCnt}}` + this.mqttClient.publish(`MqttMongo/Status/${this.options.instanceId}`, statusMsg) + this.loginfo(`Status: ${statusMsg}`) + if (! this.dbReady) { + this.emit("reconnectDatabase") + } + }, 60000) + this.loginfo("Heartbeat timer started") + } + + exec() : void { + this.connectToDatabase() + this.connectToBroker() + this.setupHeartbeat() + } +} diff --git a/src/config.ts b/src/config.ts new file mode 100644 index 0000000..70cd5cc --- /dev/null +++ b/src/config.ts @@ -0,0 +1,19 @@ +import * as fs from 'fs' +import * as cmdargs from 'command-line-args' + + + +const OPTION_DEFINITIONS = [ + { name: 'verbose', alias: 'v', type: Boolean }, + { name: 'config', alias: 'c', type: String, defaultValue: '~/MqttMongoNodejs.conf' } +]; + + +export let dict : any + +export function readConfig() { + let options = cmdargs(OPTION_DEFINITIONS) + dict = JSON.parse(fs.readFileSync(options.config, "utf8")) +} + +readConfig() diff --git a/src/log.ts b/src/log.ts new file mode 100644 index 0000000..8ab49af --- /dev/null +++ b/src/log.ts @@ -0,0 +1,52 @@ +import * as moment from 'moment' +import * as config from './config' + + +enum Level { + All, + NoDebug, + NoDebugNoInfo, + NoDebugNoInfoNoWarning +} + +var level = Level.NoDebug + +function timestamp(): string { + return moment().format('HH:mm:ss.SSS') +} + +export function setLevel(value: string): void { + switch (value) { + case 'info': level = Level.NoDebug; break + case 'warn': level = Level.NoDebugNoInfo; break + case 'error': level = Level.NoDebugNoInfoNoWarning; break + default: level = Level.All + } +} + + +export function info(message: string): void { + if (level < Level.NoDebugNoInfo) { + console.log(`${timestamp()} [ II ] ${message}`) + } +} + +export function warn(message: string): void { + if (level < Level.NoDebugNoInfoNoWarning) { + console.log(`${timestamp()} [ WW ] ${message}`) + } +} + +export function error(message: string): void { + console.log(`${timestamp()} [ EE ] ${message}`) +} + +export function success(message: string): void { + console.log(`${timestamp()} [ OK ] ${message}`) +} + +export function debug(message: string): void { + if (level < Level.NoDebug) { + console.log(`${timestamp()} [ DB ] ${message}`) + } +} diff --git a/src/main.ts b/src/main.ts index 36f5803..816c3bc 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,173 +1,32 @@ -import * as Mqtt from 'mqtt' -import * as Mongo from 'mongodb' -import * as Events from 'events' -import * as Queue from './queue' -import * as MqttMessage from './mqtt_message' +import * as log from './log' +import * as config from './config' +import MqttMongo from './MqttMongo' -let console = require('chalk-console') +let instances : MqttMongo[] = [] -const MQTT_BROKER_URL : String = 'mqtt://localhost' -const MONGO_DATABASE_URL : String = 'mongodb://localhost/test' -const COLLECTION : String = 'mqttMongo' - - -function collect(val: string, memo: string[]) { - memo.push(val); - return memo; -} - -import options = require('commander') -options - .version('0.0.1') - .option('-b, --broker [broker url]', 'Broker URL', MQTT_BROKER_URL) - .option('-m, --database [database url]', 'MongoDB Database URL', MONGO_DATABASE_URL) - .option('-c, --collection [mongodb collection]', 'Collection in MongoDB Database', COLLECTION) - .option('-t, --topic [topic to subscribe]', 'Topic to subscribe, can appear multiple times', collect, []) - .option('-e, --encapsulate', 'store current timestamp, topic and payload in document', false) - .option('-p, --parsePayload', 'parse payload when encapsulating (otherwise always)', false) - .option('-v, --verbose', 'log all inserted messages', false) - .parse(process.argv) - - -class MqttMongo extends Events.EventEmitter { - - private options : any - private msgCnt : number = 0 - private startTime: Date - - constructor(options : any) { - super() - this.options = options - this.startTime = new Date() - this.on('reconnectDatabase', this.connectToDatabase) +config.dict.instances.forEach((v: any) => { + let options : any = { + brokerUrl: config.dict.brokerUrl, + brokerUser: config.dict.brokerUser, + brokerPass: config.dict.brokerPass, + brokerCa: config.dict.brokerCa, + mongodbUrl: config.dict.mongodbUrl, + verbose: config.dict.verbose, + instanceId: v.instanceId, + collection: v.collection, + topics: v.topics, + encapsulate : v.encapsulate, + parsePayload : v.parsePayload } - private dbHandle : Mongo.Db; - private dbReady : boolean = false; - private queue : Queue.Queue = new Queue.Queue() - private mqttClient : Mqtt.Client - private heartbeatTimer : NodeJS.Timer + log.info(JSON.stringify(options)) - connectToDatabase() { - console.info("About to connect to database") - Mongo.MongoClient.connect(this.options.database) - .then( - (tmpDbHandle: Mongo.Db) => { - this.dbHandle = tmpDbHandle - this.dbReady = true; - console.info("Database connected") - this.dbHandle.on('reconnectFailed', (err : any) => { console.warn(`Error on database ${err}`) }) - this.dbHandle.on('reconnect', () => { - console.info("Reconnect on database") - this.dbReady = true - this.queue.knock() - }) - this.dbHandle.on('timeout', () => { console.warn("Timeout on database") }) - this.dbHandle.on('close', () => { - console.info("Close on database") - this.dbReady = false - }) - }, - (err: String) => { - console.error(`Unable to connect to database: ${err}`) - this.dbReady = false - } - ) - } + let instance : MqttMongo = new MqttMongo(options) + instance.exec() + instances.push(instance) +}) - connectToBroker() { - this.queue.on('data', () => { - if (this.dbReady || true) { - while (! this.queue.isEmpty()) { - let msg : MqttMessage.MqttMessage = this.queue.deq() - if (this.options.verbose) { - console.info(`Something in the queue: ${JSON.stringify(msg)}`) - } - let coll = this.dbHandle.collection(this.options.collection) - coll.insertOne(msg.getMessage()) - .then( - (r) => { - if (this.options.verbose) { - console.success(`Successfully inserted into database ${JSON.stringify(msg.getMessage())}`) - } - }, - (err) => { - console.error(`Error when trying to insert into database ${err}`) - if (! this.dbReady) { - console.info("Error occured while database connection is lost, re-enqueue msg.") - this.queue.reenq(msg) - this.emit('reconnectDatabase') - } else { - console.error(`Message ${JSON.stringify(msg.getMessage())} is lost`) - } - } - ) - } - } else { - // console.info("Database currently not available, not reading from stream") - } - }) - - this.mqttClient = Mqtt.connect(this.options.broker) - this.mqttClient.on('offline', () => { console.warn("MQTT client is offline") }) - this.mqttClient.on('reconnect', () => { console.warn("MQTT client is reconnecting") }) - this.mqttClient.on('close', () => { console.warn("MQTT connection closed") }) - this.mqttClient.on('connect', () => { - console.info("MQTT broker connected") - this.options.topic.forEach((topic: string) => { - this.mqttClient.subscribe(topic) - console.info(`Subscribed to ${topic}`) - }) - this.mqttClient.subscribe('MqttMongo/Command') - this.mqttClient.publish('MqttMongo/Status', 'hello, started up') - }) - - this.mqttClient.on('message', (topic : string, messageBuf : Buffer) => { - this.msgCnt++; - let message = messageBuf.toString('UTF-8') - if (this.options.verbose) { - console.info(`Message received ${this.msgCnt}, topic ${topic}, payload ${message}`) - } - - if (topic == "MqttMongo/Command" && message == "shutdown") { - this.shutdown() - } else { - try { - this.queue.enq(new MqttMessage.MqttMessage(topic, message, this.options.encapsulate, this.options.parsePayload)) - } catch (e) { - console.error(`Error while parsing Mqtt message, topic '${topic}', message '${message}' , ${e.toString()}`) - } - } - }) - } - - setupHeartbeat() { - this.heartbeatTimer = setInterval(() => { - let uptime : number = (new Date().getTime() - this.startTime.getTime()) / 1000 - let statusMsg = `{'Uptime': ${uptime}, 'MessageCount': ${this.msgCnt}}` - this.mqttClient.publish('MqttMongo/Status', statusMsg) - console.info(`Status: ${statusMsg}`) - if (! this.dbReady) { - this.emit("reconnectDatabase") - } - }, 60000) - console.info("Heartbeat timer started") - } - - shutdown() { - console.info("Shutting down MqttMongo") - clearInterval(this.heartbeatTimer) - this.mqttClient.end() - this.dbHandle.close() - } -} - -let mqttMongo : MqttMongo = new MqttMongo(options) -mqttMongo.connectToDatabase() -mqttMongo.connectToBroker() -mqttMongo.setupHeartbeat() - -console.info("MqttMongo started") \ No newline at end of file +log.info("MqttMongoNodejs started")