diff --git a/db/knex_migrations/2023-10-26-slow-response-notification.js b/db/knex_migrations/2023-10-26-slow-response-notification.js index 72ce236c1..8a2c19089 100644 --- a/db/knex_migrations/2023-10-26-slow-response-notification.js +++ b/db/knex_migrations/2023-10-26-slow-response-notification.js @@ -3,9 +3,11 @@ exports.up = function (knex) { return knex.schema .alterTable("monitor", function(table) { table.boolean("slow_response_notification").notNullable().defaultTo(false); - table.integer("slow_response_notification_threshold").notNullable().defaultTo(0); - table.integer("slow_response_notification_range").notNullable().defaultTo(0); table.string("slow_response_notification_method").notNullable().defaultTo(""); + table.integer("slow_response_notification_range").notNullable().defaultTo(0); + table.string("slow_response_notification_threshold_method").notNullable().defaultTo(""); + table.integer("slow_response_notification_threshold").notNullable().defaultTo(0); + table.float("slow_response_notification_threshold_multiplier").notNullable().defaultTo(0.0); table.integer("slow_response_notification_resend_interval").notNullable().defaultTo(0); }) .alterTable("heartbeat", function(table) { @@ -14,15 +16,18 @@ exports.up = function (knex) { } exports.down = function (knex) { + // remove various slow_response_notification parameters return knex.schema .alterTable("monitor", function(table) { - table.boolean("slow_response_notification").notNullable().defaultTo(false); - table.integer("slow_response_notification_threshold").notNullable().defaultTo(0); - table.integer("slow_response_notification_range").notNullable().defaultTo(0); - table.string("slow_response_notification_method").notNullable().defaultTo(""); - table.integer("slow_response_notification_resend_interval").notNullable().defaultTo(0); + table.dropColumn("slow_response_notification"); + table.dropColumn("slow_response_notification_method"); + table.dropColumn("slow_response_notification_range"); + table.dropColumn("slow_response_notification_threshold_method"); + table.dropColumn("slow_response_notification_threshold"); + table.dropColumn("slow_response_notification_threshold_multiplier"); + table.dropColumn("slow_response_notification_resend_interval"); }) .alterTable("heartbeat", function(table) { - table.integer("slow_response_count").notNullable().defaultTo(0); + table.dropColumn("slow_response_count"); }); } diff --git a/server/model/monitor.js b/server/model/monitor.js index c6ff7a83b..6678a1c8b 100644 --- a/server/model/monitor.js +++ b/server/model/monitor.js @@ -133,9 +133,11 @@ class Monitor extends BeanModel { databaseQuery: this.databaseQuery, authMethod: this.authMethod, slowResponseNotification: this.isEnabledSlowResponseNotification(), - slowResponseNotificationThreshold: this.slowResponseNotificationThreshold, - slowResponseNotificationRange: this.slowResponseNotificationRange, slowResponseNotificationMethod: this.slowResponseNotificationMethod, + slowResponseNotificationRange: this.slowResponseNotificationRange, + slowResponseNotificationThresholdMethod: this.slowResponseNotificationThresholdMethod, + slowResponseNotificationThreshold: this.slowResponseNotificationThreshold, + slowResponseNotificationThresholdMultiplier: this.slowResponseNotificationThresholdMultiplier, slowResponseNotificationResendInterval: this.slowResponseNotificationResendInterval, grpcUrl: this.grpcUrl, grpcProtobuf: this.grpcProtobuf, @@ -949,7 +951,7 @@ class Monitor extends BeanModel { bean.end_time = R.isoDateTimeMillis(endTimeDayjs); // Check if response time is slow - if (this.isEnabledSlowResponseNotification()) { + if (this.isEnabledSlowResponseNotification() && !isFirstBeat) { log.debug("monitor", `[${this.name}] Check if response is slow`); await this.checkSlowResponseNotification(this, bean); } @@ -1423,28 +1425,33 @@ class Monitor extends BeanModel { } /** - * Check heartbeat response time is slower than threshold. + * Check if heartbeat response time is slower than threshold. * @param {Monitor} monitor The monitor to send a notification about * @param {Bean} bean Status information about monitor * @returns {Promise} */ async checkSlowResponseNotification(monitor, bean) { - //Get recent heartbeat list with range of time - const afterThisDate = new Date(Date.now() - (1000 * (monitor.slowResponseNotificationRange + 1))); // add 1 second otherwise we grab 0 previous beats when Time Range == Heartbeat Interval - const previousBeats = await R.getAll(` - SELECT * FROM heartbeat - WHERE monitor_id = ? AND time > datetime(?) AND status = ?`, - [ - monitor.id, - afterThisDate.toISOString(), - UP, - ]); const method = monitor.slowResponseNotificationMethod; - const thresholdResponseTime = monitor.slowResponseNotificationThreshold; + const thresholdMethod = monitor.slowResponseNotificationThresholdMethod; + const thresholdMultipler = monitor.slowResponseNotificationThresholdMultiplier; const windowDuration = monitor.slowResponseNotificationRange; let actualResponseTime = 0; + let previousBeats; + if (method != "last") { + //Get recent heartbeat list with range of time + const afterThisDate = new Date(Date.now() - (1000 * (monitor.slowResponseNotificationRange + 1))); // add 1 second otherwise we grab 0 previous beats when Time Range == Heartbeat Interval + previousBeats = await R.getAll(` + SELECT * FROM heartbeat + WHERE monitor_id = ? AND time > datetime(?) AND status = ?`, + [ + monitor.id, + afterThisDate.toISOString(), + UP, + ]); + } + switch (method) { case "average": previousBeats.forEach(beat => { @@ -1460,58 +1467,98 @@ class Monitor extends BeanModel { break; case "last": - actualResponseTime = bean.ping + actualResponseTime = bean.ping; break; default: - log.error("monitor", `[${this.name}] Unknown slow response notification method ${method}`); + log.error("monitor", `[${this.name}] Unknown response time calculation method for slow response notification: ${method}`); + return; + } + + let threshold; + let thresholdDescription; + switch (thresholdMethod) { + case "threshold-static": + threshold = monitor.slowResponseNotificationThreshold; + thresholdDescription = "static"; + break; + + case "threshold-relative-24-hour": + //Get average response time over last 24 hours + const afterThisDate = new Date(Date.now() - (1000 * (24 * 60 * 60))); // 24 hours in milliseconds + const avgPing = parseInt(await R.getCell(` + SELECT AVG(ping) FROM heartbeat + WHERE time > datetime(?) + AND ping IS NOT NULL + AND monitor_id = ? + AND status = ? + `, + [ afterThisDate.toISOString(), monitor.id, UP ] + )); + //calculate threshold + threshold = Math.round(avgPing * thresholdMultipler); + thresholdDescription = `${thresholdMultipler}x 24H Avg`; + break; + + default: + log.error("monitor", `[${this.name}] Unknown threshold calculation method for slow response notification: ${thresholdMethod}`); return; } // Create stats to append to messages/logs - let msgStats = `\nResponse: ${actualResponseTime}ms | Threshold: ${thresholdResponseTime}ms | Method: ${method}` + const methodDescription = ["average", "max"].includes(method) ? `${method} of ${windowDuration}s` : method; + let msgStats = `Response: ${actualResponseTime}ms (${methodDescription}) | Threshold: ${threshold}ms (${thresholdDescription})` // Add window duration for methods that make sense - if (["average", "max"].includes(method)) { - msgStats += ` over ${windowDuration}s` + + // Verify valid response time was calculated + if (actualResponseTime == 0 || !Number.isInteger(actualResponseTime)) { + log.debug("monitor", `[${this.name}] Failed to calculate valid response time`); + return; } - // Verify something was actually calculated - if (actualResponseTime != 0 && Number.isInteger(actualResponseTime)) { - // Responding normally - if (actualResponseTime < thresholdResponseTime) { - if (bean.slowResponseCount == 0) { - log.debug("monitor", `[${this.name}] Responding normally. No need to send slow response notification ${msgStats}`); - } else { - log.debug("monitor", `[${this.name}] Returned to normal response time ${msgStats}`); - let msg = `[${this.name}] Returned to Normal Response Time ${msgStats}`; - Monitor.sendSlowResponseNotification(monitor, bean, msg); - } + // Verify valid threshold was calculated + if (!Number.isInteger(threshold)) { + log.debug("monitor", `[${this.name}] Failed to calculate valid threshold`); + return; + } - // Reset slow response count - bean.slowResponseCount = 0; - return; - - // Responding slowly + // Responding normally + if (actualResponseTime < threshold) { + if (bean.slowResponseCount == 0) { + log.debug("monitor", `[${this.name}] Responding normally. No need to send slow response notification | ${msgStats}`); } else { - ++bean.slowResponseCount; - - // Always send first notification - if (bean.slowResponseCount == 1) { - log.debug("monitor", `[${this.name}] Responded slowly, sending notification ${msgStats}`); - let msg = `[${this.name}] Responded Slowly ${msgStats}`; - Monitor.sendSlowResponseNotification(monitor, bean, msg); - } else if (this.slowResponseNotificationResendInterval > 0){ - // Send notification every x times - if (((bean.slowResponseCount) % this.slowResponseNotificationResendInterval) == 0) { - // Send notification again, because we are still responding slow - log.debug("monitor", `[${this.name}] sendSlowResponseNotification again ${msgStats}`); - let msg = `[${this.name}] Still Responding Slowly ${msgStats}`; - Monitor.sendSlowResponseNotification(monitor, bean, msg); - } - } + msgStats += ` | Slow for: ${bean.slowResponseCount * monitor.interval}s`; + log.debug("monitor", `[${this.name}] Returned to normal response time | ${msgStats}`); + let msg = `[${this.name}] Returned to Normal Response Time \n${msgStats}`; + Monitor.sendSlowResponseNotification(monitor, bean, msg); } + + // Reset slow response count + bean.slowResponseCount = 0; + + // Responding slowly } else { - log.debug("monitor", `[${this.name}] Failed to calculate valid response time`); + ++bean.slowResponseCount; + + // Always send first notification + if (bean.slowResponseCount == 1) { + log.debug("monitor", `[${this.name}] Responded slowly, sending notification | ${msgStats}`); + let msg = `[${this.name}] Responded Slowly \n${msgStats}`; + Monitor.sendSlowResponseNotification(monitor, bean, msg); + // Send notification every x times + } else if (this.slowResponseNotificationResendInterval > 0){ + if (((bean.slowResponseCount) % this.slowResponseNotificationResendInterval) == 0) { + // Send notification again, because we are still responding slow + msgStats += ` | Slow for: ${bean.slowResponseCount * monitor.interval}s`; + log.debug("monitor", `[${this.name}] Still responding slowly, sendSlowResponseNotification again | ${msgStats}`); + let msg = `[${this.name}] Still Responding Slowly \n${msgStats}`; + Monitor.sendSlowResponseNotification(monitor, bean, msg); + } else { + log.debug("monitor", `[${this.name}] Still responding slowly, waiting for resend interal | ${msgStats}`); + } + } else { + log.debug("monitor", `[${this.name}] Still responding slowly, but resend is disabled | ${msgStats}`); + } } } diff --git a/server/server.js b/server/server.js index 2fe30e608..6f55ef888 100644 --- a/server/server.js +++ b/server/server.js @@ -804,9 +804,11 @@ let needSetup = false; bean.authWorkstation = monitor.authWorkstation; bean.authDomain = monitor.authDomain; bean.slowResponseNotification = monitor.slowResponseNotification; - bean.slowResponseNotificationThreshold = monitor.slowResponseNotificationThreshold; - bean.slowResponseNotificationRange = monitor.slowResponseNotificationRange; bean.slowResponseNotificationMethod = monitor.slowResponseNotificationMethod; + bean.slowResponseNotificationRange = monitor.slowResponseNotificationRange; + bean.slowResponseNotificationThresholdMethod = monitor.slowResponseNotificationThresholdMethod; + bean.slowResponseNotificationThreshold = monitor.slowResponseNotificationThreshold; + bean.slowResponseNotificationThresholdMultiplier = monitor.slowResponseNotificationThresholdMultiplier; bean.slowResponseNotificationResendInterval = monitor.slowResponseNotificationResendInterval; bean.grpcUrl = monitor.grpcUrl; bean.grpcProtobuf = monitor.grpcProtobuf; diff --git a/src/lang/en.json b/src/lang/en.json index e0a0e6c78..a5d0cb11c 100644 --- a/src/lang/en.json +++ b/src/lang/en.json @@ -491,9 +491,16 @@ "slowResponseNotificationUseDescription": "Send a notification when service response time is slow.", "slowResponseNotificationThreshold": "Threshold (ms)", "slowResponseNotificationThresholdDescription": "Send a notification if calculated response time is greater than {0} ms.", + "slowResponseNotificationThresholdMethod": "Threshold Calculation", + "slowResponseNotificationThresholdMethodStatic": "Static Threshold", + "slowResponseNotificationThresholdMethodStaticDescription": "Define a static threshold.", + "slowResponseNotificationThresholdMethodRelative24Hour": "Relative to Avg. Response", + "slowResponseNotificationThresholdMethodRelative24HourDescription": "Calculate the threshold ({0}x the 24-hour average response time).", + "slowResponseNotificationThresholdMultiplier": "Threshold Multiplier", + "slowResponseNotificationThresholdMultiplierDescription": "Send notification if response time is greater than {0}x the 24-hour average.", "slowResponseNotificationRange": "Window Duration (seconds)", "slowResponseNotificationRangeDescription": "Window duration for calculating the {0}.", - "slowResponseNotificationMethod": "Calculation Method", + "slowResponseNotificationMethod": "Response Time Calculation", "slowResponseNotificationMethodAverage": "Average", "slowResponseNotificationMethodAverageDescription": "Get the average response time over the last {0} seconds.", "slowResponseNotificationMethodMax": "Max", diff --git a/src/pages/EditMonitor.vue b/src/pages/EditMonitor.vue index 0ab307422..eb420a609 100644 --- a/src/pages/EditMonitor.vue +++ b/src/pages/EditMonitor.vue @@ -426,8 +426,8 @@
- - @@ -449,14 +449,7 @@
-
- - -
- {{ $t("slowResponseNotificationThresholdDescription", [monitor.slowResponseNotificationThreshold]) }} -
-
- +
@@ -465,6 +458,44 @@
+ +
+ + +
+ {{ $t("slowResponseNotificationThresholdMethodStaticDescription") }} +
+
+ {{ $t("slowResponseNotificationThresholdMethodRelative24HourDescription", [monitor.slowResponseNotificationThresholdMultiplier]) }} +
+
+ + +
+ + +
+ {{ $t("slowResponseNotificationThresholdDescription", [monitor.slowResponseNotificationThreshold]) }} +
+
+ + +
+ + +
+ {{ $t("slowResponseNotificationThresholdMultiplierDescription", [monitor.slowResponseNotificationThresholdMultiplier]) }} +
+
+ +