Clean up notification logic and messages

This commit is contained in:
Stephen Papierski 2023-11-07 15:51:59 -07:00
parent 9c5bb42b20
commit 51e48e6054
4 changed files with 55 additions and 49 deletions

View file

@ -1404,26 +1404,21 @@ class Monitor extends BeanModel {
* Send a slow response notification about a monitor * Send a slow response notification about a monitor
* @param {Monitor} monitor The monitor to send a notificaton about * @param {Monitor} monitor The monitor to send a notificaton about
* @param {Bean} bean Status information about monitor * @param {Bean} bean Status information about monitor
* @param {string} message Notification text to be sent * @param {string} msg Notification text to be sent
* @returns {void} * @returns {void}
*/ */
static async sendSlowResponseNotification(monitor, bean, message) { static async sendSlowResponseNotification(monitor, bean, msg) {
// Send notification // Send notification
const notificationList = await Monitor.getNotificationList(monitor); const notificationList = await Monitor.getNotificationList(monitor);
if (notificationList.length > 0) { for (let notification of notificationList) {
for (let notification of notificationList) { try {
try { log.debug("monitor", `[${this.name}] Sending to ${notification.name}`);
log.debug("monitor", `[${this.name}] Sending to ${notification.name}`); await Notification.send(JSON.parse(notification.config), msg);
await Notification.send(JSON.parse(notification.config), message); } catch (e) {
} catch (e) { log.error("monitor", `[${this.name}] Cannot send slow response notification to ${notification.name}`);
log.error("monitor", `[${this.name}] Cannot send slow response notification to ${notification.name}`); log.error("monitor", e);
log.error("monitor", e);
}
} }
} else {
log.debug("monitor", `[${this.name}] No notification configured, no need to send slow response notification`);
} }
} }
@ -1447,6 +1442,7 @@ class Monitor extends BeanModel {
]); ]);
const method = monitor.slowResponseNotificationMethod; const method = monitor.slowResponseNotificationMethod;
const thresholdResponseTime = monitor.slowResponseNotificationThreshold; const thresholdResponseTime = monitor.slowResponseNotificationThreshold;
const windowDuration = monitor.slowResponseNotificationRange;
let actualResponseTime = 0; let actualResponseTime = 0;
switch (method) { switch (method) {
@ -1472,39 +1468,50 @@ class Monitor extends BeanModel {
return; return;
} }
// Responding normally // Create stats to append to messages/logs
if (actualResponseTime < thresholdResponseTime) { let msgStats = `\nResponse: ${actualResponseTime}ms | Threshold: ${thresholdResponseTime}ms | Method: ${method}`
if (bean.slowResponseCount == 0) { // Add window duration for methods that make sense
log.debug("monitor", `[${this.name}] Responding normally. No need to send slow response notification (${actualResponseTime}ms < ${thresholdResponseTime}ms)`); if (["average", "max"].includes(method)) {
msgStats += ` over ${windowDuration}s`
}
// Verify something was actually calculated
if (actualResponseTime != 0 && Number.isInteger(actualResponseTime)) {
// Responding normally
if (actualResponseTime < thresholdResponseTime) {
if (bean.slowResponseCount == 0) {
log.debug("monitor", `[${this.name}] Responding normally. No need to send slow response notification ${msgStats}`);
} else {
log.debug("monitor", `[${this.name}] Returned to normal response time ${msgStats}`);
let msg = `[${this.name}] Returned to Normal Response Time ${msgStats}`;
Monitor.sendSlowResponseNotification(monitor, bean, msg);
}
// Reset slow response count
bean.slowResponseCount = 0;
return;
// Responding slowly
} else { } else {
log.debug("monitor", `[${this.name}] Returned to normal response time (${actualResponseTime}ms < ${thresholdResponseTime}ms)`); ++bean.slowResponseCount;
let message = `[${this.name}] Returned to normal response time (${actualResponseTime}ms < ${thresholdResponseTime}ms)`;
Monitor.sendSlowResponseNotification(monitor, bean, message);
}
// Reset slow response count // Always send first notification
bean.slowResponseCount = 0; if (bean.slowResponseCount == 1) {
return; log.debug("monitor", `[${this.name}] Responded slowly, sending notification ${msgStats}`);
let msg = `[${this.name}] Responded Slowly ${msgStats}`;
// Responding slowly Monitor.sendSlowResponseNotification(monitor, bean, msg);
} else { } else if (this.slowResponseNotificationResendInterval > 0){
++bean.slowResponseCount; // Send notification every x times
log.debug("monitor", `[${this.name}] Responded slowly (${actualResponseTime}ms > ${thresholdResponseTime}ms, Slow Response Count: ${bean.slowResponseCount})`); if (((bean.slowResponseCount) % this.slowResponseNotificationResendInterval) == 0) {
// Send notification again, because we are still responding slow
// Always send first notification log.debug("monitor", `[${this.name}] sendSlowResponseNotification again ${msgStats}`);
if (bean.slowResponseCount == 1) { let msg = `[${this.name}] Still Responding Slowly ${msgStats}`;
log.debug("monitor", `[${this.name}] Responded slowly, sending notification (${actualResponseTime}ms > ${thresholdResponseTime}ms, Slow Response Count: ${bean.slowResponseCount})`); Monitor.sendSlowResponseNotification(monitor, bean, msg);
let message = `[${this.name}] Started responding slowly (${actualResponseTime}ms > ${thresholdResponseTime}ms)`; }
Monitor.sendSlowResponseNotification(monitor, bean, message);
} else if (this.slowResponseNotificationResendInterval > 0){
// Send notification every x times
if (((bean.slowResponseCount) % this.slowResponseNotificationResendInterval) == 0) {
// Send notification again, because we are still responding slow
log.debug("monitor", `[${this.name}] sendSlowResponseNotification again (${actualResponseTime}ms > ${thresholdResponseTime}ms, Slow Response Count: ${bean.slowResponseCount})`);
let message = `[${this.name}] Still responding slowly (${actualResponseTime}ms > ${thresholdResponseTime}ms, Slow Response Count: ${bean.slowResponseCount})`;
Monitor.sendSlowResponseNotification(monitor, bean, message);
} }
} }
} else {
log.debug("monitor", `[${this.name}] Failed to calculate valid response time`);
} }
} }

View file

@ -43,8 +43,7 @@ log.debug("server", "Arguments");
log.debug("server", args); log.debug("server", args);
if (! process.env.NODE_ENV) { if (! process.env.NODE_ENV) {
// process.env.NODE_ENV = "production"; process.env.NODE_ENV = "production";
process.env.NODE_ENV = "development";
} }
log.info("server", "Env: " + process.env.NODE_ENV); log.info("server", "Env: " + process.env.NODE_ENV);

View file

@ -490,9 +490,9 @@
"slowResponseNotificationEnable": "Slow Response Notification", "slowResponseNotificationEnable": "Slow Response Notification",
"slowResponseNotificationUseDescription": "Send a notification when service response time is slow.", "slowResponseNotificationUseDescription": "Send a notification when service response time is slow.",
"slowResponseNotificationThreshold": "Threshold (ms)", "slowResponseNotificationThreshold": "Threshold (ms)",
"slowResponseNotificationThresholdDescription": "Send a notification if response time is greater than {0} ms.", "slowResponseNotificationThresholdDescription": "Send a notification if calculated response time is greater than {0} ms.",
"slowResponseNotificationRange": "Time Range (seconds)", "slowResponseNotificationRange": "Window Duration (seconds)",
"slowResponseNotificationRangeDescription": "Gets the heartbeat information for the last {0} seconds and calculates the condition.", "slowResponseNotificationRangeDescription": "Window duration for calculating the {0}.",
"slowResponseNotificationMethod": "Calculation Method", "slowResponseNotificationMethod": "Calculation Method",
"slowResponseNotificationMethodAverage": "Average", "slowResponseNotificationMethodAverage": "Average",
"slowResponseNotificationMethodAverageDescription": "Get the average response time over the last {0} seconds.", "slowResponseNotificationMethodAverageDescription": "Get the average response time over the last {0} seconds.",

View file

@ -461,7 +461,7 @@
<label for="slow-response-notification-range" class="form-label">{{ $t("slowResponseNotificationRange") }}</label> <label for="slow-response-notification-range" class="form-label">{{ $t("slowResponseNotificationRange") }}</label>
<input id="slow-response-notification-range" v-model="monitor.slowResponseNotificationRange" type="number" class="form-control" required :min="monitor.interval" step="1"> <input id="slow-response-notification-range" v-model="monitor.slowResponseNotificationRange" type="number" class="form-control" required :min="monitor.interval" step="1">
<div class="form-text"> <div class="form-text">
{{ $t("slowResponseNotificationRangeDescription", [monitor.slowResponseNotificationRange]) }} {{ $t("slowResponseNotificationRangeDescription", [monitor.slowResponseNotificationMethod]) }}
</div> </div>
</div> </div>