Add slow response notification resend interval, also notify when response time returns to normal

db/knex_migrations/2023-10-26-slow-response-notification.js

@@ -2,21 +2,27 @@ exports.up = function (knex) {
     // add various slow_response_notification parameters
     return knex.schema
         .alterTable("monitor", function(table) {
-            table.boolean("slow_response_notification").defaultTo(false);
+            table.boolean("slow_response_notification").notNullable().defaultTo(false);
-            table.integer("slow_response_notification_threshold").defaultTo(0);
+            table.integer("slow_response_notification_threshold").notNullable().defaultTo(0);
-            table.integer("slow_response_notification_range").defaultTo(0);
+            table.integer("slow_response_notification_range").notNullable().defaultTo(0);
-            table.string("slow_response_notification_method").defaultTo("");
+            table.string("slow_response_notification_method").notNullable().defaultTo("");
-            table.integer("slow_response_notification_resend_interval").defaultTo(0);
+            table.integer("slow_response_notification_resend_interval").notNullable().defaultTo(0);
+        })
+        .alterTable("heartbeat", function(table) {
+            table.integer("slow_response_count").notNullable().defaultTo(0);
         });
 }
 
 exports.down = function (knex) {
     return knex.schema
         .alterTable("monitor", function(table) {
-            table.boolean("slow_response_notification").defaultTo(false);
+            table.boolean("slow_response_notification").notNullable().defaultTo(false);
-            table.integer("slow_response_notification_threshold").defaultTo(0);
+            table.integer("slow_response_notification_threshold").notNullable().defaultTo(0);
-            table.integer("slow_response_notification_range").defaultTo(0);
+            table.integer("slow_response_notification_range").notNullable().defaultTo(0);
-            table.string("slow_response_notification_method").defaultTo("");
+            table.string("slow_response_notification_method").notNullable().defaultTo("");
-            table.integer("slow_response_notification_resend_interval").defaultTo(0);
+            table.integer("slow_response_notification_resend_interval").notNullable().defaultTo(0);
+        })
+        .alterTable("heartbeat", function(table) {
+            table.integer("slow_response_count").notNullable().defaultTo(0);
         });
 }

server/model/monitor.js

@@ -136,6 +136,7 @@ class Monitor extends BeanModel {
             slowResponseNotificationThreshold: this.slowResponseNotificationThreshold,
             slowResponseNotificationRange: this.slowResponseNotificationRange,
             slowResponseNotificationMethod: this.slowResponseNotificationMethod,
+            slowResponseNotificationResendInterval: this.slowResponseNotificationResendInterval,
             grpcUrl: this.grpcUrl,
             grpcProtobuf: this.grpcProtobuf,
             grpcMethod: this.grpcMethod,
@@ -353,6 +354,7 @@ class Monitor extends BeanModel {
             bean.time = R.isoDateTimeMillis(dayjs.utc());
             bean.status = DOWN;
             bean.downCount = previousBeat?.downCount || 0;
+            bean.slowResponseCount = previousBeat?.slowResponseCount || 0;
 
             if (this.isUpsideDown()) {
                 bean.status = flipStatus(bean.status);
@@ -941,6 +943,12 @@ class Monitor extends BeanModel {
             let endTimeDayjs = await uptimeCalculator.update(bean.status, parseFloat(bean.ping));
             bean.end_time = R.isoDateTimeMillis(endTimeDayjs);
 
+            // Check if response time is slow
+            if (this.isEnabledSlowResponseNotification()) {
+                log.debug("monitor", `[${this.name}] Check if response is slow`);
+                await this.checkSlowResponseNotification(this, bean);
+            }
+
             // Send to frontend
             log.debug("monitor", `[${this.name}] Send to socket`);
             io.to(this.user_id).emit("heartbeat", bean.toJSON());
@@ -950,11 +958,6 @@ class Monitor extends BeanModel {
             log.debug("monitor", `[${this.name}] Store`);
             await R.store(bean);
 
-            if (this.isEnabledSlowResponseNotification()) {
-                log.debug("monitor", `[${this.name}] Check response is slow`);
-                await this.checkSlowResponseNotification(this);
-            }
-
             log.debug("monitor", `[${this.name}] prometheus.update`);
             this.prometheus?.update(bean, tlsInfo);
 
@@ -1392,12 +1395,40 @@ class Monitor extends BeanModel {
         }
     }
 
+    /**
+     * Send a slow response notification about a monitor
+     * @param {Monitor} monitor The monitor to send a notificaton about
+     * @param {Bean} bean Status information about monitor
+     * @param {string} message Notification text to be sent
+     * @returns {void}
+     */
+    static async sendSlowResponseNotification(monitor, bean, message) {
+        // Send notification
+        const notificationList = await Monitor.getNotificationList(monitor);
+
+        if (notificationList.length > 0) {
+            for (let notification of notificationList) {
+                try {
+                    log.debug("monitor", `[${this.name}] Sending to ${notification.name}`);
+                    await Notification.send(JSON.parse(notification.config), message);
+                } catch (e) {
+                    log.error("monitor", `[${this.name}] Cannot send slow response notification to ${notification.name}`);
+                    log.error("monitor", e);
+                }
+            }
+
+        } else {
+            log.debug("monitor", `[${this.name}] No notification configured, no need to send slow response notification`);
+        }
+    }
+
     /**
      * Check heartbeat response time is slower than threshold.
      * @param {Monitor} monitor The monitor to send a notification about
+     * @param {Bean} bean Status information about monitor
      * @returns {Promise<void>}
      */
-    async checkSlowResponseNotification(monitor) {
+    async checkSlowResponseNotification(monitor, bean) {
 
         //Get recent heartbeat list with range of time
         const afterThisDate = new Date(Date.now() - (1000 * monitor.slowResponseNotificationRange));
@@ -1432,28 +1463,39 @@ class Monitor extends BeanModel {
                 return;
         }
 
+        // Responding normally
         if (actualResponseTime < thresholdResponseTime) {
-            log.debug("monitor", `[${this.name}] No need to send slow notification. ${actualResponseTime} < ${thresholdResponseTime}`);
+            if (bean.slowResponseCount == 0) {
-            return;
+                log.debug("monitor", `[${this.name}] Responding normally. No need to send slow response notification (${actualResponseTime}ms < ${thresholdResponseTime}ms)`);
-        }
+            } else {
-
+                log.debug("monitor", `[${this.name}] Returned to normal response time (${actualResponseTime}ms < ${thresholdResponseTime}ms)`);
-        log.debug("monitor", `[${this.name}] Try to send slow response notification (${actualResponseTime} > ${thresholdResponseTime})`);
+                let message = `[${this.name}] Returned to normal response time (${actualResponseTime}ms < ${thresholdResponseTime}ms)`;
-
+                Monitor.sendSlowResponseNotification(monitor, bean, message);
-        const notificationList = await Monitor.getNotificationList(monitor);
-
-        if (notificationList.length > 0) {
-            for (let notification of notificationList) {
-                try {
-                    log.debug("monitor", `[${this.name}] Sending to ${notification.name}`);
-                    await Notification.send(JSON.parse(notification.config), `[${this.name}] Responding slowly (${actualResponseTime}ms > ${thresholdResponseTime}ms)`);
-                } catch (e) {
-                    log.error("monitor", `[${this.name}] Cannot send slow response notification to ${notification.name}`);
-                    log.error("monitor", e);
-                }
             }
 
+            // Reset slow response count
+            bean.slowResponseCount = 0;
+            return;
+
+        // Responding slowly
         } else {
-            log.debug("monitor", `[${this.name}] No notification, no need to send slow response notification`);
+            ++bean.slowResponseCount;
+            log.debug("monitor", `[${this.name}] Responded slowly (${actualResponseTime}ms > ${thresholdResponseTime}ms, Slow Response Count: ${bean.slowResponseCount})`);
+
+            // Always send first notification
+            if (this.slowResponseNotificationResendInterval <= 0 || bean.slowResponseCount == 1) {
+                log.debug("monitor", `[${this.name}] Responded slowly, sending notification (${actualResponseTime}ms > ${thresholdResponseTime}ms, Slow Response Count: ${bean.slowResponseCount})`);
+                let message = `[${this.name}] Started responding slowly (${actualResponseTime}ms > ${thresholdResponseTime}ms)`;
+                Monitor.sendSlowResponseNotification(monitor, bean, message);
+            } else {
+                // Send notification every x times
+                if (((bean.slowResponseCount) % this.slowResponseNotificationResendInterval) == 0) {
+                    // Send notification again, because we are still responding slow
+                    log.debug("monitor", `[${this.name}] sendSlowResponseNotification again (${actualResponseTime}ms > ${thresholdResponseTime}ms, Slow Response Count: ${bean.slowResponseCount})`);
+                    let message = `[${this.name}] Still responding slowly (${actualResponseTime}ms > ${thresholdResponseTime}ms, Slow Response Count: ${bean.slowResponseCount})`;
+                    Monitor.sendSlowResponseNotification(monitor, bean, message);
+                }
+            }
         }
     }
 

server/server.js

@@ -808,6 +808,7 @@ let needSetup = false;
                 bean.slowResponseNotificationThreshold = monitor.slowResponseNotificationThreshold;
                 bean.slowResponseNotificationRange = monitor.slowResponseNotificationRange;
                 bean.slowResponseNotificationMethod = monitor.slowResponseNotificationMethod;
+                bean.slowResponseNotificationResendInterval = monitor.slowResponseNotificationResendInterval;
                 bean.grpcUrl = monitor.grpcUrl;
                 bean.grpcProtobuf = monitor.grpcProtobuf;
                 bean.grpcServiceName = monitor.grpcServiceName;

src/lang/en.json

@@ -485,18 +485,18 @@
     "uninstall": "Uninstall",
     "uninstalling": "Uninstalling",
     "confirmUninstallPlugin": "Are you sure want to uninstall this plugin?",
-   "slowResponseNotification": "Slow Response Notification",
+    "slowResponseNotificationEnable": "Slow Response Notification",
-    "slowResponseNotificationUse": "Use Slow Response Notification",
+    "slowResponseNotificationUseDescription": "Send a notification when service response time is slow.",
-    "slowResponseNotificationUseDescription": "When response time is slow, notify.",
     "slowResponseNotificationThreshold": "Threshold (ms)",
     "slowResponseNotificationThresholdDescription": "If response time greater than {0} ms, notify.",
     "slowResponseNotificationRange": "Time Range (seconds)",
     "slowResponseNotificationRangeDescription": "Gets the heartbeat information for the last {0} seconds and calculates the condition.",
     "slowResponseNotificationMethod": "Calculation Method",
     "slowResponseNotificationMethodAverage": "Average",
-    "slowResponseNotificationMethodAverageDescription": "Get the average response time of the last {0} seconds.",
+    "slowResponseNotificationMethodAverageDescription": "Get the average response time over the last {0} seconds.",
     "slowResponseNotificationMethodMax": "Max",
     "slowResponseNotificationMethodMaxDescription": "Get the max response time of the last {0} seconds.",
+    "slowResponseNotificationResendInterval": "Resend Notification if Slow Response X times consecutively",
     "notificationRegional": "Regional",
     "Clone Monitor": "Clone Monitor",
     "Clone": "Clone",

src/pages/EditMonitor.vue

@@ -418,12 +418,11 @@
                                 <input id="resend-interval" v-model="monitor.resendInterval" type="number" class="form-control" required min="0" step="1">
                             </div>
 
-                            <h2 class="mt-5 mb-2">{{ $t("slowResponseNotification") }}</h2>
+                            <!-- Slow Response Notification -->
-
                             <div class="my-3 form-check">
                                 <input id="slow-response-notification" v-model="monitor.slowResponseNotification" class="form-check-input" type="checkbox">
                                 <label class="form-check-label" for="slow-response-notification">
-                                    {{ $t("slowResponseNotificationUse") }}
+                                    {{ $t("slowResponseNotificationEnable") }}
                                 </label>
                                 <div class="form-text">
                                     {{ $t("slowResponseNotificationUseDescription") }}
@@ -467,7 +466,7 @@
 
                             <div v-if="monitor.slowResponseNotification" class="my-3">
                                 <label for="slow-response-notification-resend-interval" class="form-label">
-                                    {{ $t("Resend Notification if Slow Response X times consecutively") }}
+                                    {{ $t("slowResponseNotificationResendInterval", [monitor.slowResponseNotificationInterval]) }}
                                     <span v-if="monitor.slowResponseNotificationResendInterval > 0">({{ $t("resendEveryXTimes", [ monitor.slowResponseNotificationResendInterval ]) }})</span>
                                     <span v-else>({{ $t("resendDisabled") }})</span>
                                 </label>