improve page load performance of large amount urls (#5025)

Co-authored-by: vishal sabhaya <vishals@vebuin.com>
Co-authored-by: Frank Elsinga <frank@elsinga.de>

server/model/monitor.js

@@ -72,23 +72,12 @@ class Monitor extends BeanModel {
 
     /**
      * Return an object that ready to parse to JSON
+     * @param {object} preloadData to prevent n+1 problems, we query the data in a batch outside of this function
      * @param {boolean} includeSensitiveData Include sensitive data in
      * JSON
-     * @returns {Promise<object>} Object ready to parse
+     * @returns {object} Object ready to parse
      */
-    async toJSON(includeSensitiveData = true) {
+    toJSON(preloadData = {}, includeSensitiveData = true) {
-
-        let notificationIDList = {};
-
-        let list = await R.find("monitor_notification", " monitor_id = ? ", [
-            this.id,
-        ]);
-
-        for (let bean of list) {
-            notificationIDList[bean.notification_id] = true;
-        }
-
-        const tags = await this.getTags();
 
         let screenshot = null;
 
@@ -96,7 +85,7 @@ class Monitor extends BeanModel {
             screenshot = "/screenshots/" + jwt.sign(this.id, UptimeKumaServer.getInstance().jwtSecret) + ".png";
         }
 
-        const path = await this.getPath();
+        const path = preloadData.paths.get(this.id) || [];
         const pathName = path.join(" / ");
 
         let data = {
@@ -106,15 +95,15 @@ class Monitor extends BeanModel {
             path,
             pathName,
             parent: this.parent,
-            childrenIDs: await Monitor.getAllChildrenIDs(this.id),
+            childrenIDs: preloadData.childrenIDs.get(this.id) || [],
             url: this.url,
             method: this.method,
             hostname: this.hostname,
             port: this.port,
             maxretries: this.maxretries,
             weight: this.weight,
-            active: await this.isActive(),
+            active: preloadData.activeStatus.get(this.id),
-            forceInactive: !await Monitor.isParentActive(this.id),
+            forceInactive: preloadData.forceInactive.get(this.id),
             type: this.type,
             timeout: this.timeout,
             interval: this.interval,
@@ -134,9 +123,9 @@ class Monitor extends BeanModel {
             docker_container: this.docker_container,
             docker_host: this.docker_host,
             proxyId: this.proxy_id,
-            notificationIDList,
+            notificationIDList: preloadData.notifications.get(this.id) || {},
-            tags: tags,
+            tags: preloadData.tags.get(this.id) || [],
-            maintenance: await Monitor.isUnderMaintenance(this.id),
+            maintenance: preloadData.maintenanceStatus.get(this.id),
             mqttTopic: this.mqttTopic,
             mqttSuccessMessage: this.mqttSuccessMessage,
             mqttCheckType: this.mqttCheckType,
@@ -202,16 +191,6 @@ class Monitor extends BeanModel {
         return data;
     }
 
-    /**
-     * Checks if the monitor is active based on itself and its parents
-     * @returns {Promise<boolean>} Is the monitor active?
-     */
-    async isActive() {
-        const parentActive = await Monitor.isParentActive(this.id);
-
-        return (this.active === 1) && parentActive;
-    }
-
     /**
      * Get all tags applied to this monitor
      * @returns {Promise<LooseObject<any>[]>} List of tags on the
@@ -1197,6 +1176,18 @@ class Monitor extends BeanModel {
         return checkCertificateResult;
     }
 
+    /**
+     * Checks if the monitor is active based on itself and its parents
+     * @param {number} monitorID ID of monitor to send
+     * @param {boolean} active is active
+     * @returns {Promise<boolean>} Is the monitor active?
+     */
+    static async isActive(monitorID, active) {
+        const parentActive = await Monitor.isParentActive(monitorID);
+
+        return (active === 1) && parentActive;
+    }
+
     /**
      * Send statistics to clients
      * @param {Server} io Socket server instance
@@ -1333,7 +1324,10 @@ class Monitor extends BeanModel {
             for (let notification of notificationList) {
                 try {
                     const heartbeatJSON = bean.toJSON();
-
+                    const monitorData = [{ id: monitor.id,
+                        active: monitor.active
+                    }];
+                    const preloadData = await Monitor.preparePreloadData(monitorData);
                     // Prevent if the msg is undefined, notifications such as Discord cannot send out.
                     if (!heartbeatJSON["msg"]) {
                         heartbeatJSON["msg"] = "N/A";
@@ -1344,7 +1338,7 @@ class Monitor extends BeanModel {
                     heartbeatJSON["timezoneOffset"] = UptimeKumaServer.getInstance().getTimezoneOffset();
                     heartbeatJSON["localDateTime"] = dayjs.utc(heartbeatJSON["time"]).tz(heartbeatJSON["timezone"]).format(SQL_DATETIME_FORMAT);
 
-                    await Notification.send(JSON.parse(notification.config), msg, await monitor.toJSON(false), heartbeatJSON);
+                    await Notification.send(JSON.parse(notification.config), msg, monitor.toJSON(preloadData, false), heartbeatJSON);
                 } catch (e) {
                     log.error("monitor", "Cannot send notification to " + notification.name);
                     log.error("monitor", e);
@@ -1507,6 +1501,111 @@ class Monitor extends BeanModel {
         }
     }
 
+    /**
+     * Gets monitor notification of multiple monitor
+     * @param {Array} monitorIDs IDs of monitor to get
+     * @returns {Promise<LooseObject<any>>} object
+     */
+    static async getMonitorNotification(monitorIDs) {
+        return await R.getAll(`
+            SELECT monitor_notification.monitor_id, monitor_notification.notification_id
+            FROM monitor_notification
+            WHERE monitor_notification.monitor_id IN (?)
+        `, [
+            monitorIDs,
+        ]);
+    }
+
+    /**
+     * Gets monitor tags of multiple monitor
+     * @param {Array} monitorIDs IDs of monitor to get
+     * @returns {Promise<LooseObject<any>>} object
+     */
+    static async getMonitorTag(monitorIDs) {
+        return await R.getAll(`
+            SELECT monitor_tag.monitor_id, tag.name, tag.color
+            FROM monitor_tag
+            JOIN tag ON monitor_tag.tag_id = tag.id
+            WHERE monitor_tag.monitor_id IN (?)
+        `, [
+            monitorIDs,
+        ]);
+    }
+
+    /**
+     * prepare preloaded data for efficient access
+     * @param {Array} monitorData IDs & active field of monitor to get
+     * @returns {Promise<LooseObject<any>>} object
+     */
+    static async preparePreloadData(monitorData) {
+
+        const notificationsMap = new Map();
+        const tagsMap = new Map();
+        const maintenanceStatusMap = new Map();
+        const childrenIDsMap = new Map();
+        const activeStatusMap = new Map();
+        const forceInactiveMap = new Map();
+        const pathsMap = new Map();
+
+        if (monitorData.length > 0) {
+            const monitorIDs = monitorData.map(monitor => monitor.id);
+            const notifications = await Monitor.getMonitorNotification(monitorIDs);
+            const tags = await Monitor.getMonitorTag(monitorIDs);
+            const maintenanceStatuses = await Promise.all(monitorData.map(monitor => Monitor.isUnderMaintenance(monitor.id)));
+            const childrenIDs = await Promise.all(monitorData.map(monitor => Monitor.getAllChildrenIDs(monitor.id)));
+            const activeStatuses = await Promise.all(monitorData.map(monitor => Monitor.isActive(monitor.id, monitor.active)));
+            const forceInactiveStatuses = await Promise.all(monitorData.map(monitor => Monitor.isParentActive(monitor.id)));
+            const paths = await Promise.all(monitorData.map(monitor => Monitor.getAllPath(monitor.id, monitor.name)));
+
+            notifications.forEach(row => {
+                if (!notificationsMap.has(row.monitor_id)) {
+                    notificationsMap.set(row.monitor_id, {});
+                }
+                notificationsMap.get(row.monitor_id)[row.notification_id] = true;
+            });
+
+            tags.forEach(row => {
+                if (!tagsMap.has(row.monitor_id)) {
+                    tagsMap.set(row.monitor_id, []);
+                }
+                tagsMap.get(row.monitor_id).push({
+                    name: row.name,
+                    color: row.color
+                });
+            });
+
+            monitorData.forEach((monitor, index) => {
+                maintenanceStatusMap.set(monitor.id, maintenanceStatuses[index]);
+            });
+
+            monitorData.forEach((monitor, index) => {
+                childrenIDsMap.set(monitor.id, childrenIDs[index]);
+            });
+
+            monitorData.forEach((monitor, index) => {
+                activeStatusMap.set(monitor.id, activeStatuses[index]);
+            });
+
+            monitorData.forEach((monitor, index) => {
+                forceInactiveMap.set(monitor.id, !forceInactiveStatuses[index]);
+            });
+
+            monitorData.forEach((monitor, index) => {
+                pathsMap.set(monitor.id, paths[index]);
+            });
+        }
+
+        return {
+            notifications: notificationsMap,
+            tags: tagsMap,
+            maintenanceStatus: maintenanceStatusMap,
+            childrenIDs: childrenIDsMap,
+            activeStatus: activeStatusMap,
+            forceInactive: forceInactiveMap,
+            paths: pathsMap,
+        };
+    }
+
     /**
      * Gets Parent of the monitor
      * @param {number} monitorID ID of monitor to get
@@ -1539,16 +1638,18 @@ class Monitor extends BeanModel {
 
     /**
      * Gets the full path
+     * @param {number} monitorID ID of the monitor to get
+     * @param {string} name of the monitor to get
      * @returns {Promise<string[]>} Full path (includes groups and the name) of the monitor
      */
-    async getPath() {
+    static async getAllPath(monitorID, name) {
-        const path = [ this.name ];
+        const path = [ name ];
 
         if (this.parent === null) {
             return path;
         }
 
-        let parent = await Monitor.getParent(this.id);
+        let parent = await Monitor.getParent(monitorID);
         while (parent !== null) {
             path.unshift(parent.name);
             parent = await Monitor.getParent(parent.id);

server/server.js

@@ -726,7 +726,7 @@ let needSetup = false;
 
                 await updateMonitorNotification(bean.id, notificationIDList);
 
-                await server.sendMonitorList(socket);
+                await server.sendUpdateMonitorIntoList(socket, bean.id);
 
                 if (monitor.active !== false) {
                     await startMonitor(socket.userID, bean.id);
@@ -879,11 +879,11 @@ let needSetup = false;
 
                 await updateMonitorNotification(bean.id, monitor.notificationIDList);
 
-                if (await bean.isActive()) {
+                if (await Monitor.isActive(bean.id, bean.active)) {
                     await restartMonitor(socket.userID, bean.id);
                 }
 
-                await server.sendMonitorList(socket);
+                await server.sendUpdateMonitorIntoList(socket, bean.id);
 
                 callback({
                     ok: true,
@@ -923,14 +923,17 @@ let needSetup = false;
 
                 log.info("monitor", `Get Monitor: ${monitorID} User ID: ${socket.userID}`);
 
-                let bean = await R.findOne("monitor", " id = ? AND user_id = ? ", [
+                let monitor = await R.findOne("monitor", " id = ? AND user_id = ? ", [
                     monitorID,
                     socket.userID,
                 ]);
-
+                const monitorData = [{ id: monitor.id,
+                    active: monitor.active
+                }];
+                const preloadData = await Monitor.preparePreloadData(monitorData);
                 callback({
                     ok: true,
-                    monitor: await bean.toJSON(),
+                    monitor: monitor.toJSON(preloadData),
                 });
 
             } catch (e) {
@@ -981,7 +984,7 @@ let needSetup = false;
             try {
                 checkLogin(socket);
                 await startMonitor(socket.userID, monitorID);
-                await server.sendMonitorList(socket);
+                await server.sendUpdateMonitorIntoList(socket, monitorID);
 
                 callback({
                     ok: true,
@@ -1001,7 +1004,7 @@ let needSetup = false;
             try {
                 checkLogin(socket);
                 await pauseMonitor(socket.userID, monitorID);
-                await server.sendMonitorList(socket);
+                await server.sendUpdateMonitorIntoList(socket, monitorID);
 
                 callback({
                     ok: true,
@@ -1047,8 +1050,7 @@ let needSetup = false;
                     msg: "successDeleted",
                     msgi18n: true,
                 });
-
+                await server.sendDeleteMonitorFromList(socket, monitorID);
-                await server.sendMonitorList(socket);
 
             } catch (e) {
                 callback({
@@ -1678,13 +1680,13 @@ async function afterLogin(socket, user) {
 
     await StatusPage.sendStatusPageList(io, socket);
 
+    const monitorPromises = [];
     for (let monitorID in monitorList) {
-        await sendHeartbeatList(socket, monitorID);
+        monitorPromises.push(sendHeartbeatList(socket, monitorID));
+        monitorPromises.push(Monitor.sendStats(io, monitorID, user.id));
     }
 
-    for (let monitorID in monitorList) {
+    await Promise.all(monitorPromises);
-        await Monitor.sendStats(io, monitorID, user.id);
-    }
 
     // Set server timezone from client browser if not set
     // It should be run once only

server/uptime-kuma-server.js

@@ -205,24 +205,56 @@ class UptimeKumaServer {
         return list;
     }
 
+    /**
+     * Update Monitor into list
+     * @param {Socket} socket Socket to send list on
+     * @param {number} monitorID update or deleted monitor id
+     * @returns {Promise<void>}
+     */
+    async sendUpdateMonitorIntoList(socket, monitorID) {
+        let list = await this.getMonitorJSONList(socket.userID, monitorID);
+        this.io.to(socket.userID).emit("updateMonitorIntoList", list);
+    }
+
+    /**
+     * Delete Monitor from list
+     * @param {Socket} socket Socket to send list on
+     * @param {number} monitorID update or deleted monitor id
+     * @returns {Promise<void>}
+     */
+    async sendDeleteMonitorFromList(socket, monitorID) {
+        this.io.to(socket.userID).emit("deleteMonitorFromList", monitorID);
+    }
+
     /**
      * Get a list of monitors for the given user.
      * @param {string} userID - The ID of the user to get monitors for.
+     * @param {number} monitorID - The ID of monitor for.
      * @returns {Promise<object>} A promise that resolves to an object with monitor IDs as keys and monitor objects as values.
      *
      * Generated by Trelent
      */
-    async getMonitorJSONList(userID) {
+    async getMonitorJSONList(userID, monitorID = null) {
-        let result = {};
 
-        let monitorList = await R.find("monitor", " user_id = ? ORDER BY weight DESC, name", [
+        let query = " user_id = ? ";
-            userID,
+        let queryParams = [ userID ];
-        ]);
 
-        for (let monitor of monitorList) {
+        if (monitorID) {
-            result[monitor.id] = await monitor.toJSON();
+            query += "AND id = ? ";
+            queryParams.push(monitorID);
         }
 
+        let monitorList = await R.find("monitor", query + "ORDER BY weight DESC, name", queryParams);
+
+        const monitorData = monitorList.map(monitor => ({
+            id: monitor.id,
+            active: monitor.active,
+            name: monitor.name,
+        }));
+        const preloadData = await Monitor.preparePreloadData(monitorData);
+
+        const result = {};
+        monitorList.forEach(monitor => result[monitor.id] = monitor.toJSON(preloadData));
         return result;
     }
 
@@ -520,3 +552,4 @@ const { DnsMonitorType } = require("./monitor-types/dns");
 const { MqttMonitorType } = require("./monitor-types/mqtt");
 const { SNMPMonitorType } = require("./monitor-types/snmp");
 const { MongodbMonitorType } = require("./monitor-types/mongodb");
+const Monitor = require("./model/monitor");

src/mixins/socket.js

@@ -141,19 +141,23 @@ export default {
             });
 
             socket.on("monitorList", (data) => {
-                // Add Helper function
+                this.assignMonitorUrlParser(data);
-                Object.entries(data).forEach(([ monitorID, monitor ]) => {
-                    monitor.getUrl = () => {
-                        try {
-                            return new URL(monitor.url);
-                        } catch (_) {
-                            return null;
-                        }
-                    };
-                });
                 this.monitorList = data;
             });
 
+            socket.on("updateMonitorIntoList", (data) => {
+                this.assignMonitorUrlParser(data);
+                Object.entries(data).forEach(([ monitorID, updatedMonitor ]) => {
+                    this.monitorList[monitorID] = updatedMonitor;
+                });
+            });
+
+            socket.on("deleteMonitorFromList", (monitorID) => {
+                if (this.monitorList[monitorID]) {
+                    delete this.monitorList[monitorID];
+                }
+            });
+
             socket.on("monitorTypeList", (data) => {
                 this.monitorTypeList = data;
             });
@@ -289,6 +293,23 @@ export default {
                 location.reload();
             });
         },
+        /**
+         * parse all urls from list.
+         * @param {object} data Monitor data to modify
+         * @returns {object} list
+         */
+        assignMonitorUrlParser(data) {
+            Object.entries(data).forEach(([ monitorID, monitor ]) => {
+                monitor.getUrl = () => {
+                    try {
+                        return new URL(monitor.url);
+                    } catch (_) {
+                        return null;
+                    }
+                };
+            });
+            return data;
+        },
 
         /**
          * The storage currently in use

src/pages/EditMonitor.vue

@@ -1813,7 +1813,6 @@ message HealthCheckResponse {
                             await this.startParentGroupMonitor();
                         }
                         this.processing = false;
-                        this.$root.getMonitorList();
                         this.$router.push("/dashboard/" + res.monitorID);
                     } else {
                         this.processing = false;

src/util.js

@@ -83,6 +83,7 @@ exports.CONSOLE_STYLE_BgMagenta = "\x1b[45m";
 exports.CONSOLE_STYLE_BgCyan = "\x1b[46m";
 exports.CONSOLE_STYLE_BgWhite = "\x1b[47m";
 exports.CONSOLE_STYLE_BgGray = "\x1b[100m";
+
 const consoleModuleColors = [
     exports.CONSOLE_STYLE_FgCyan,
     exports.CONSOLE_STYLE_FgGreen,