From 8fc0b6a0a180138cc4a30fbc1be5d20dc8dbf2b0 Mon Sep 17 00:00:00 2001 From: Stephen Papierski Date: Tue, 31 Oct 2023 23:17:35 -0600 Subject: [PATCH] Add slow response notification resend interval, also notify when response time returns to normal --- .../2023-10-26-slow-response-notification.js | 26 +++--- server/model/monitor.js | 90 ++++++++++++++----- server/server.js | 1 + src/lang/en.json | 8 +- src/pages/EditMonitor.vue | 7 +- 5 files changed, 90 insertions(+), 42 deletions(-) diff --git a/db/knex_migrations/2023-10-26-slow-response-notification.js b/db/knex_migrations/2023-10-26-slow-response-notification.js index 892703962..72ce236c1 100644 --- a/db/knex_migrations/2023-10-26-slow-response-notification.js +++ b/db/knex_migrations/2023-10-26-slow-response-notification.js @@ -2,21 +2,27 @@ exports.up = function (knex) { // add various slow_response_notification parameters return knex.schema .alterTable("monitor", function(table) { - table.boolean("slow_response_notification").defaultTo(false); - table.integer("slow_response_notification_threshold").defaultTo(0); - table.integer("slow_response_notification_range").defaultTo(0); - table.string("slow_response_notification_method").defaultTo(""); - table.integer("slow_response_notification_resend_interval").defaultTo(0); + table.boolean("slow_response_notification").notNullable().defaultTo(false); + table.integer("slow_response_notification_threshold").notNullable().defaultTo(0); + table.integer("slow_response_notification_range").notNullable().defaultTo(0); + table.string("slow_response_notification_method").notNullable().defaultTo(""); + table.integer("slow_response_notification_resend_interval").notNullable().defaultTo(0); + }) + .alterTable("heartbeat", function(table) { + table.integer("slow_response_count").notNullable().defaultTo(0); }); } exports.down = function (knex) { return knex.schema .alterTable("monitor", function(table) { - table.boolean("slow_response_notification").defaultTo(false); - table.integer("slow_response_notification_threshold").defaultTo(0); - table.integer("slow_response_notification_range").defaultTo(0); - table.string("slow_response_notification_method").defaultTo(""); - table.integer("slow_response_notification_resend_interval").defaultTo(0); + table.boolean("slow_response_notification").notNullable().defaultTo(false); + table.integer("slow_response_notification_threshold").notNullable().defaultTo(0); + table.integer("slow_response_notification_range").notNullable().defaultTo(0); + table.string("slow_response_notification_method").notNullable().defaultTo(""); + table.integer("slow_response_notification_resend_interval").notNullable().defaultTo(0); + }) + .alterTable("heartbeat", function(table) { + table.integer("slow_response_count").notNullable().defaultTo(0); }); } diff --git a/server/model/monitor.js b/server/model/monitor.js index 60f641158..eb6041f0d 100644 --- a/server/model/monitor.js +++ b/server/model/monitor.js @@ -136,6 +136,7 @@ class Monitor extends BeanModel { slowResponseNotificationThreshold: this.slowResponseNotificationThreshold, slowResponseNotificationRange: this.slowResponseNotificationRange, slowResponseNotificationMethod: this.slowResponseNotificationMethod, + slowResponseNotificationResendInterval: this.slowResponseNotificationResendInterval, grpcUrl: this.grpcUrl, grpcProtobuf: this.grpcProtobuf, grpcMethod: this.grpcMethod, @@ -353,6 +354,7 @@ class Monitor extends BeanModel { bean.time = R.isoDateTimeMillis(dayjs.utc()); bean.status = DOWN; bean.downCount = previousBeat?.downCount || 0; + bean.slowResponseCount = previousBeat?.slowResponseCount || 0; if (this.isUpsideDown()) { bean.status = flipStatus(bean.status); @@ -941,6 +943,12 @@ class Monitor extends BeanModel { let endTimeDayjs = await uptimeCalculator.update(bean.status, parseFloat(bean.ping)); bean.end_time = R.isoDateTimeMillis(endTimeDayjs); + // Check if response time is slow + if (this.isEnabledSlowResponseNotification()) { + log.debug("monitor", `[${this.name}] Check if response is slow`); + await this.checkSlowResponseNotification(this, bean); + } + // Send to frontend log.debug("monitor", `[${this.name}] Send to socket`); io.to(this.user_id).emit("heartbeat", bean.toJSON()); @@ -950,11 +958,6 @@ class Monitor extends BeanModel { log.debug("monitor", `[${this.name}] Store`); await R.store(bean); - if (this.isEnabledSlowResponseNotification()) { - log.debug("monitor", `[${this.name}] Check response is slow`); - await this.checkSlowResponseNotification(this); - } - log.debug("monitor", `[${this.name}] prometheus.update`); this.prometheus?.update(bean, tlsInfo); @@ -1392,12 +1395,40 @@ class Monitor extends BeanModel { } } + /** + * Send a slow response notification about a monitor + * @param {Monitor} monitor The monitor to send a notificaton about + * @param {Bean} bean Status information about monitor + * @param {string} message Notification text to be sent + * @returns {void} + */ + static async sendSlowResponseNotification(monitor, bean, message) { + // Send notification + const notificationList = await Monitor.getNotificationList(monitor); + + if (notificationList.length > 0) { + for (let notification of notificationList) { + try { + log.debug("monitor", `[${this.name}] Sending to ${notification.name}`); + await Notification.send(JSON.parse(notification.config), message); + } catch (e) { + log.error("monitor", `[${this.name}] Cannot send slow response notification to ${notification.name}`); + log.error("monitor", e); + } + } + + } else { + log.debug("monitor", `[${this.name}] No notification configured, no need to send slow response notification`); + } + } + /** * Check heartbeat response time is slower than threshold. * @param {Monitor} monitor The monitor to send a notification about + * @param {Bean} bean Status information about monitor * @returns {Promise} */ - async checkSlowResponseNotification(monitor) { + async checkSlowResponseNotification(monitor, bean) { //Get recent heartbeat list with range of time const afterThisDate = new Date(Date.now() - (1000 * monitor.slowResponseNotificationRange)); @@ -1432,28 +1463,39 @@ class Monitor extends BeanModel { return; } + // Responding normally if (actualResponseTime < thresholdResponseTime) { - log.debug("monitor", `[${this.name}] No need to send slow notification. ${actualResponseTime} < ${thresholdResponseTime}`); - return; - } - - log.debug("monitor", `[${this.name}] Try to send slow response notification (${actualResponseTime} > ${thresholdResponseTime})`); - - const notificationList = await Monitor.getNotificationList(monitor); - - if (notificationList.length > 0) { - for (let notification of notificationList) { - try { - log.debug("monitor", `[${this.name}] Sending to ${notification.name}`); - await Notification.send(JSON.parse(notification.config), `[${this.name}] Responding slowly (${actualResponseTime}ms > ${thresholdResponseTime}ms)`); - } catch (e) { - log.error("monitor", `[${this.name}] Cannot send slow response notification to ${notification.name}`); - log.error("monitor", e); - } + if (bean.slowResponseCount == 0) { + log.debug("monitor", `[${this.name}] Responding normally. No need to send slow response notification (${actualResponseTime}ms < ${thresholdResponseTime}ms)`); + } else { + log.debug("monitor", `[${this.name}] Returned to normal response time (${actualResponseTime}ms < ${thresholdResponseTime}ms)`); + let message = `[${this.name}] Returned to normal response time (${actualResponseTime}ms < ${thresholdResponseTime}ms)`; + Monitor.sendSlowResponseNotification(monitor, bean, message); } + // Reset slow response count + bean.slowResponseCount = 0; + return; + + // Responding slowly } else { - log.debug("monitor", `[${this.name}] No notification, no need to send slow response notification`); + ++bean.slowResponseCount; + log.debug("monitor", `[${this.name}] Responded slowly (${actualResponseTime}ms > ${thresholdResponseTime}ms, Slow Response Count: ${bean.slowResponseCount})`); + + // Always send first notification + if (this.slowResponseNotificationResendInterval <= 0 || bean.slowResponseCount == 1) { + log.debug("monitor", `[${this.name}] Responded slowly, sending notification (${actualResponseTime}ms > ${thresholdResponseTime}ms, Slow Response Count: ${bean.slowResponseCount})`); + let message = `[${this.name}] Started responding slowly (${actualResponseTime}ms > ${thresholdResponseTime}ms)`; + Monitor.sendSlowResponseNotification(monitor, bean, message); + } else { + // Send notification every x times + if (((bean.slowResponseCount) % this.slowResponseNotificationResendInterval) == 0) { + // Send notification again, because we are still responding slow + log.debug("monitor", `[${this.name}] sendSlowResponseNotification again (${actualResponseTime}ms > ${thresholdResponseTime}ms, Slow Response Count: ${bean.slowResponseCount})`); + let message = `[${this.name}] Still responding slowly (${actualResponseTime}ms > ${thresholdResponseTime}ms, Slow Response Count: ${bean.slowResponseCount})`; + Monitor.sendSlowResponseNotification(monitor, bean, message); + } + } } } diff --git a/server/server.js b/server/server.js index acb38b8a3..c49ccd355 100644 --- a/server/server.js +++ b/server/server.js @@ -808,6 +808,7 @@ let needSetup = false; bean.slowResponseNotificationThreshold = monitor.slowResponseNotificationThreshold; bean.slowResponseNotificationRange = monitor.slowResponseNotificationRange; bean.slowResponseNotificationMethod = monitor.slowResponseNotificationMethod; + bean.slowResponseNotificationResendInterval = monitor.slowResponseNotificationResendInterval; bean.grpcUrl = monitor.grpcUrl; bean.grpcProtobuf = monitor.grpcProtobuf; bean.grpcServiceName = monitor.grpcServiceName; diff --git a/src/lang/en.json b/src/lang/en.json index 5be3b3033..2a42f711e 100644 --- a/src/lang/en.json +++ b/src/lang/en.json @@ -485,18 +485,18 @@ "uninstall": "Uninstall", "uninstalling": "Uninstalling", "confirmUninstallPlugin": "Are you sure want to uninstall this plugin?", - "slowResponseNotification": "Slow Response Notification", - "slowResponseNotificationUse": "Use Slow Response Notification", - "slowResponseNotificationUseDescription": "When response time is slow, notify.", + "slowResponseNotificationEnable": "Slow Response Notification", + "slowResponseNotificationUseDescription": "Send a notification when service response time is slow.", "slowResponseNotificationThreshold": "Threshold (ms)", "slowResponseNotificationThresholdDescription": "If response time greater than {0} ms, notify.", "slowResponseNotificationRange": "Time Range (seconds)", "slowResponseNotificationRangeDescription": "Gets the heartbeat information for the last {0} seconds and calculates the condition.", "slowResponseNotificationMethod": "Calculation Method", "slowResponseNotificationMethodAverage": "Average", - "slowResponseNotificationMethodAverageDescription": "Get the average response time of the last {0} seconds.", + "slowResponseNotificationMethodAverageDescription": "Get the average response time over the last {0} seconds.", "slowResponseNotificationMethodMax": "Max", "slowResponseNotificationMethodMaxDescription": "Get the max response time of the last {0} seconds.", + "slowResponseNotificationResendInterval": "Resend Notification if Slow Response X times consecutively", "notificationRegional": "Regional", "Clone Monitor": "Clone Monitor", "Clone": "Clone", diff --git a/src/pages/EditMonitor.vue b/src/pages/EditMonitor.vue index eebb52dc1..c68f8cde6 100644 --- a/src/pages/EditMonitor.vue +++ b/src/pages/EditMonitor.vue @@ -418,12 +418,11 @@ -

{{ $t("slowResponseNotification") }}

- +
{{ $t("slowResponseNotificationUseDescription") }} @@ -467,7 +466,7 @@