diff --git a/db/knex_migrations/2023-10-26-slow-response-notification.js b/db/knex_migrations/2023-10-26-slow-response-notification.js new file mode 100644 index 000000000..0c33ff993 --- /dev/null +++ b/db/knex_migrations/2023-10-26-slow-response-notification.js @@ -0,0 +1,20 @@ +exports.up = function (knex) { + // add various slow_response_notification parameters + return knex.schema + .alterTable("monitor", function(table) { + table.boolean("slow_response_notification").defaultTo(false); + table.integer("slow_response_notification_threshold").defaultTo(0); + table.integer("slow_response_notification_range").defaultTo(0); + table.string("slow_response_notification_method").defaultTo(""); + }); +} + +exports.down = function (knex) { + return knex.schema + .alterTable("monitor", function(table) { + table.boolean("slow_response_notification").defaultTo(false); + table.integer("slow_response_notification_threshold").defaultTo(0); + table.integer("slow_response_notification_range").defaultTo(0); + table.string("slow_response_notification_method").defaultTo(""); + }); +} diff --git a/server/model/monitor.js b/server/model/monitor.js index 5dcb7171c..60f641158 100644 --- a/server/model/monitor.js +++ b/server/model/monitor.js @@ -132,6 +132,10 @@ class Monitor extends BeanModel { mqttSuccessMessage: this.mqttSuccessMessage, databaseQuery: this.databaseQuery, authMethod: this.authMethod, + slowResponseNotification: this.isEnabledSlowResponseNotification(), + slowResponseNotificationThreshold: this.slowResponseNotificationThreshold, + slowResponseNotificationRange: this.slowResponseNotificationRange, + slowResponseNotificationMethod: this.slowResponseNotificationMethod, grpcUrl: this.grpcUrl, grpcProtobuf: this.grpcProtobuf, grpcMethod: this.grpcMethod, @@ -298,6 +302,14 @@ class Monitor extends BeanModel { return Boolean(this.gamedigGivenPortOnly); } + /** + * Is the slow response notification enabled? + * @returns {boolean} + */ + isEnabledSlowResponseNotification() { + return Boolean(this.slowResponseNotification); + } + /** * Start monitor * @param {Server} io Socket server instance @@ -938,6 +950,11 @@ class Monitor extends BeanModel { log.debug("monitor", `[${this.name}] Store`); await R.store(bean); + if (this.isEnabledSlowResponseNotification()) { + log.debug("monitor", `[${this.name}] Check response is slow`); + await this.checkSlowResponseNotification(this); + } + log.debug("monitor", `[${this.name}] prometheus.update`); this.prometheus?.update(bean, tlsInfo); @@ -1375,6 +1392,71 @@ class Monitor extends BeanModel { } } + /** + * Check heartbeat response time is slower than threshold. + * @param {Monitor} monitor The monitor to send a notification about + * @returns {Promise} + */ + async checkSlowResponseNotification(monitor) { + + //Get recent heartbeat list with range of time + const afterThisDate = new Date(Date.now() - (1000 * monitor.slowResponseNotificationRange)); + const previousBeats = await R.getAll(` + SELECT * FROM heartbeat + WHERE monitor_id = ? AND time > datetime(?) AND status = ?`, + [ + monitor.id, + afterThisDate.toISOString(), + UP, + ]); + const method = monitor.slowResponseNotificationMethod; + const thresholdResponseTime = monitor.slowResponseNotificationThreshold; + let actualResponseTime = 0; + + switch (method) { + case "average": + previousBeats.forEach(beat => { + actualResponseTime = actualResponseTime + beat.ping; + }); + actualResponseTime = actualResponseTime / previousBeats.length; + break; + + case "max": + previousBeats.forEach(beat => { + actualResponseTime = Math.max(actualResponseTime, beat.ping); + }); + break; + + default: + log.error("monitor", `[${this.name}] Unknown slow response notification method ${method}`); + return; + } + + if (actualResponseTime < thresholdResponseTime) { + log.debug("monitor", `[${this.name}] No need to send slow notification. ${actualResponseTime} < ${thresholdResponseTime}`); + return; + } + + log.debug("monitor", `[${this.name}] Try to send slow response notification (${actualResponseTime} > ${thresholdResponseTime})`); + + const notificationList = await Monitor.getNotificationList(monitor); + + if (notificationList.length > 0) { + for (let notification of notificationList) { + try { + log.debug("monitor", `[${this.name}] Sending to ${notification.name}`); + await Notification.send(JSON.parse(notification.config), `[${this.name}] Responding slowly (${actualResponseTime}ms > ${thresholdResponseTime}ms)`); + } catch (e) { + log.error("monitor", `[${this.name}] Cannot send slow response notification to ${notification.name}`); + log.error("monitor", e); + } + } + + } else { + log.debug("monitor", `[${this.name}] No notification, no need to send slow response notification`); + } + } + /** * Get the status of the previous heartbeat * @param {number} monitorID ID of monitor to check diff --git a/server/server.js b/server/server.js index f726790c2..acb38b8a3 100644 --- a/server/server.js +++ b/server/server.js @@ -43,7 +43,8 @@ log.debug("server", "Arguments"); log.debug("server", args); if (! process.env.NODE_ENV) { - process.env.NODE_ENV = "production"; + // process.env.NODE_ENV = "production"; + process.env.NODE_ENV = "development"; } log.info("server", "Env: " + process.env.NODE_ENV); @@ -803,6 +804,10 @@ let needSetup = false; bean.authMethod = monitor.authMethod; bean.authWorkstation = monitor.authWorkstation; bean.authDomain = monitor.authDomain; + bean.slowResponseNotification = monitor.slowResponseNotification; + bean.slowResponseNotificationThreshold = monitor.slowResponseNotificationThreshold; + bean.slowResponseNotificationRange = monitor.slowResponseNotificationRange; + bean.slowResponseNotificationMethod = monitor.slowResponseNotificationMethod; bean.grpcUrl = monitor.grpcUrl; bean.grpcProtobuf = monitor.grpcProtobuf; bean.grpcServiceName = monitor.grpcServiceName; diff --git a/src/lang/en.json b/src/lang/en.json index 057b2790b..5be3b3033 100644 --- a/src/lang/en.json +++ b/src/lang/en.json @@ -485,6 +485,18 @@ "uninstall": "Uninstall", "uninstalling": "Uninstalling", "confirmUninstallPlugin": "Are you sure want to uninstall this plugin?", + "slowResponseNotification": "Slow Response Notification", + "slowResponseNotificationUse": "Use Slow Response Notification", + "slowResponseNotificationUseDescription": "When response time is slow, notify.", + "slowResponseNotificationThreshold": "Threshold (ms)", + "slowResponseNotificationThresholdDescription": "If response time greater than {0} ms, notify.", + "slowResponseNotificationRange": "Time Range (seconds)", + "slowResponseNotificationRangeDescription": "Gets the heartbeat information for the last {0} seconds and calculates the condition.", + "slowResponseNotificationMethod": "Calculation Method", + "slowResponseNotificationMethodAverage": "Average", + "slowResponseNotificationMethodAverageDescription": "Get the average response time of the last {0} seconds.", + "slowResponseNotificationMethodMax": "Max", + "slowResponseNotificationMethodMaxDescription": "Get the max response time of the last {0} seconds.", "notificationRegional": "Regional", "Clone Monitor": "Clone Monitor", "Clone": "Clone", diff --git a/src/pages/EditMonitor.vue b/src/pages/EditMonitor.vue index aef964233..78e7417a5 100644 --- a/src/pages/EditMonitor.vue +++ b/src/pages/EditMonitor.vue @@ -418,6 +418,53 @@ +

{{ $t("slowResponseNotification") }}

+ +
+ + +
+ {{ $t("slowResponseNotificationUseDescription") }} +
+
+ + +
+ + +
+ {{ $t("slowResponseNotificationMethodAverageDescription", [monitor.slowResponseNotificationRange]) }} +
+
+ {{ $t("slowResponseNotificationMethodMaxDescription", [monitor.slowResponseNotificationRange]) }} +
+
+ +
+ + +
+ {{ $t("slowResponseNotificationThresholdDescription", [monitor.slowResponseNotificationThreshold]) }} +
+
+ +
+ + +
+ {{ $t("slowResponseNotificationRangeDescription", [monitor.slowResponseNotificationRange]) }} +
+
+

{{ $t("Advanced") }}

@@ -888,6 +935,10 @@ const monitorDefaults = { }, kafkaProducerSsl: false, gamedigGivenPortOnly: true, + slowResponseNotification: false, + slowResponseNotificationThreshold: 5000, + slowResponseNotificationRange: 60, + slowResponseNotificationMethod: "average", }; export default {