From 2dfa6886b44fe642039d9896057f33528a2146a5 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Sun, 1 Sep 2024 17:19:18 +0800 Subject: [PATCH 01/19] Preparing --- server/database.js | 9 +++++++++ server/jobs/clear-old-data.js | 11 +++++++++++ 2 files changed, 20 insertions(+) diff --git a/server/database.js b/server/database.js index 3374aff9e..76d1139ee 100644 --- a/server/database.js +++ b/server/database.js @@ -711,6 +711,15 @@ class Database { } } + /** + * TODO: Migrate the old data in the heartbeat table to the new format (stat_daily, stat_hourly, stat_minutely) + * It should be run once while upgrading V1 to V2 + * @returns {Promise} + */ + static async migrateAggregateTable() { + + } + } module.exports = Database; diff --git a/server/jobs/clear-old-data.js b/server/jobs/clear-old-data.js index 248a4d409..d74f34845 100644 --- a/server/jobs/clear-old-data.js +++ b/server/jobs/clear-old-data.js @@ -11,6 +11,17 @@ const DEFAULT_KEEP_PERIOD = 180; */ const clearOldData = async () => { + + /* + * TODO: + * Since we have aggregated table now, we don't need so much data in heartbeat table. + * But we still need to keep the important rows, because they contain the message. + * + * In the heartbeat table: + * - important rows: keep according to the setting (keepDataPeriodDays) (default 180 days) + * - not important rows: keep 2 days + */ + let period = await setting("keepDataPeriodDays"); // Set Default Period From 124effb55211fa1fbbfc0319f98f99a442cf065e Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Sun, 22 Sep 2024 16:01:45 +0800 Subject: [PATCH 02/19] wip --- server/jobs/clear-old-data.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/jobs/clear-old-data.js b/server/jobs/clear-old-data.js index d74f34845..5bbd48a92 100644 --- a/server/jobs/clear-old-data.js +++ b/server/jobs/clear-old-data.js @@ -20,6 +20,9 @@ const clearOldData = async () => { * In the heartbeat table: * - important rows: keep according to the setting (keepDataPeriodDays) (default 180 days) * - not important rows: keep 2 days + * + * stat_* tables: + * - keep according to the setting (keepDataPeriodDays) (default 180 days) */ let period = await setting("keepDataPeriodDays"); From 0f3c727aa4669f05fe1237abc4b7978a1d189c01 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Sun, 22 Sep 2024 18:39:46 +0800 Subject: [PATCH 03/19] wip and fix sqlite migration because of foreign_keys --- server/database.js | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/server/database.js b/server/database.js index 76d1139ee..317c21662 100644 --- a/server/database.js +++ b/server/database.js @@ -6,6 +6,7 @@ const knex = require("knex"); const path = require("path"); const { EmbeddedMariaDB } = require("./embedded-mariadb"); const mysql = require("mysql2/promise"); +const { Settings } = require("./settings"); /** * Database & App Data Folder @@ -391,9 +392,23 @@ class Database { // https://knexjs.org/guide/migrations.html // https://gist.github.com/NigelEarle/70db130cc040cc2868555b29a0278261 try { + // Disable foreign key check for SQLite + // Known issue of knex: https://github.com/drizzle-team/drizzle-orm/issues/1813 + if (Database.dbConfig.type === "sqlite") { + await R.exec("PRAGMA foreign_keys = OFF"); + } + await R.knex.migrate.latest({ directory: Database.knexMigrationsPath, }); + + // Enable foreign key check for SQLite + if (Database.dbConfig.type === "sqlite") { + await R.exec("PRAGMA foreign_keys = ON"); + } + + await this.migrateAggregateTable(); + } catch (e) { // Allow missing patch files for downgrade or testing pr. if (e.message.includes("the following files are missing:")) { @@ -717,7 +732,37 @@ class Database { * @returns {Promise} */ static async migrateAggregateTable() { + log.debug("db", "Enter Migrate Aggregate Table function"); + // + let migrated = false; + + if (migrated) { + log.debug("db", "Migrated, skip migration"); + return; + } + + log.info("db", "Migrating Aggregate Table"); + + // Migrate heartbeat to stat_minutely, using knex transaction + const trx = await R.knex.transaction(); + + // Get a list of unique dates from the heartbeat table, using raw sql + let dates = await trx.raw(` + SELECT DISTINCT DATE(time) AS date + FROM heartbeat + `); + + // Get a list of unique monitors from the heartbeat table, using raw sql + let monitors = await trx.raw(` + SELECT DISTINCT monitor_id + FROM heartbeat + `); + + console.log("Dates", dates); + console.log("Monitors", monitors); + + trx.commit(); } } From 59e7607e1adab0e5dc7396a9b1734273c81c2299 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Fri, 27 Sep 2024 21:42:20 +0800 Subject: [PATCH 04/19] wip --- server/database.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/server/database.js b/server/database.js index 317c21662..79ef091b0 100644 --- a/server/database.js +++ b/server/database.js @@ -735,7 +735,7 @@ class Database { log.debug("db", "Enter Migrate Aggregate Table function"); // - let migrated = false; + let migrated = Settings.get("migratedAggregateTable"); if (migrated) { log.debug("db", "Migrated, skip migration"); @@ -759,10 +759,20 @@ class Database { FROM heartbeat `); + // Show warning if stat_* tables are not empty + for (let table of [ "stat_minutely", "stat_hourly", "stat_daily" ]) { + let count = await trx(table).count("*").first(); + if (count.count > 0) { + log.warn("db", `Table ${table} is not empty, migration may cause data loss (Maybe you were using 2.0.0-dev?)`); + } + } + console.log("Dates", dates); console.log("Monitors", monitors); trx.commit(); + + //await Settings.set("migratedAggregateTable", true); } } From 67ad0f79b3c995bc1f07469ab4758727b0cd33aa Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Sat, 28 Sep 2024 01:22:45 +0800 Subject: [PATCH 05/19] wip --- server/database.js | 37 ++++++++++++++++++++++++++++++++----- server/uptime-calculator.js | 7 +++++-- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/server/database.js b/server/database.js index 4a6ac7433..55dce6a47 100644 --- a/server/database.js +++ b/server/database.js @@ -6,6 +6,8 @@ const path = require("path"); const { EmbeddedMariaDB } = require("./embedded-mariadb"); const mysql = require("mysql2/promise"); const { Settings } = require("./settings"); +const { UptimeCalculator } = require("./uptime-calculator"); +const dayjs = require("dayjs"); /** * Database & App Data Folder @@ -734,7 +736,7 @@ class Database { log.debug("db", "Enter Migrate Aggregate Table function"); // - let migrated = Settings.get("migratedAggregateTable"); + let migrated = await Settings.get("migratedAggregateTable"); if (migrated) { log.debug("db", "Migrated, skip migration"); @@ -750,6 +752,7 @@ class Database { let dates = await trx.raw(` SELECT DISTINCT DATE(time) AS date FROM heartbeat + ORDER BY date ASC `); // Get a list of unique monitors from the heartbeat table, using raw sql @@ -758,17 +761,41 @@ class Database { FROM heartbeat `); - // Show warning if stat_* tables are not empty + // Stop if stat_* tables are not empty for (let table of [ "stat_minutely", "stat_hourly", "stat_daily" ]) { - let count = await trx(table).count("*").first(); - if (count.count > 0) { - log.warn("db", `Table ${table} is not empty, migration may cause data loss (Maybe you were using 2.0.0-dev?)`); + let countResult = await trx.raw(`SELECT COUNT(*) AS count FROM ${table}`); + let count = countResult[0].count; + if (count > 0) { + log.warn("db", `Aggregate table ${table} is not empty, migration will not be started (Maybe you were using 2.0.0-dev?)`); + return; } } console.log("Dates", dates); console.log("Monitors", monitors); + for (let monitor of monitors) { + for (let date of dates) { + log.info("db", `Migrating monitor ${monitor.monitor_id} on date ${date.date}`); + + // New Uptime Calculator + let calculator = new UptimeCalculator(); + + // TODO: Pass transaction to the calculator + // calculator.setTransaction(trx); + + // Get all the heartbeats for this monitor and date + let heartbeats = await trx("heartbeat") + .where("monitor_id", monitor.monitor_id) + .whereRaw("DATE(time) = ?", [ date.date ]) + .orderBy("time", "asc"); + + for (let heartbeat of heartbeats) { + calculator.update(heartbeat.status, heartbeat.ping, dayjs(heartbeat.time)); + } + } + } + trx.commit(); //await Settings.set("migratedAggregateTable", true); diff --git a/server/uptime-calculator.js b/server/uptime-calculator.js index f2738b96a..e9ade777e 100644 --- a/server/uptime-calculator.js +++ b/server/uptime-calculator.js @@ -189,11 +189,14 @@ class UptimeCalculator { /** * @param {number} status status * @param {number} ping Ping + * @param {dayjs.Dayjs} date Date (Only for migration) * @returns {dayjs.Dayjs} date * @throws {Error} Invalid status */ - async update(status, ping = 0) { - let date = this.getCurrentDate(); + async update(status, ping = 0, date) { + if (!date) { + date = this.getCurrentDate(); + } let flatStatus = this.flatStatus(status); From 31ce34da771fb6372e87e8416fbb0fe10f3310bc Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Mon, 7 Oct 2024 18:56:07 +0800 Subject: [PATCH 06/19] wip --- server/database.js | 1 + 1 file changed, 1 insertion(+) diff --git a/server/database.js b/server/database.js index 55dce6a47..75093b2f5 100644 --- a/server/database.js +++ b/server/database.js @@ -767,6 +767,7 @@ class Database { let count = countResult[0].count; if (count > 0) { log.warn("db", `Aggregate table ${table} is not empty, migration will not be started (Maybe you were using 2.0.0-dev?)`); + trx.commit(); return; } } From da8da0bf596bbbc7290800f9d1b4154e58f019ac Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Tue, 8 Oct 2024 18:47:56 +0800 Subject: [PATCH 07/19] wip --- server/database.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/server/database.js b/server/database.js index 75093b2f5..a82eaa813 100644 --- a/server/database.js +++ b/server/database.js @@ -735,6 +735,12 @@ class Database { static async migrateAggregateTable() { log.debug("db", "Enter Migrate Aggregate Table function"); + // Add a setting for 2.0.0-dev users to skip this migration + if (process.env.SET_MIGRATE_AGGREGATE_TABLE_TO_TRUE === "1") { + log.warn("db", "SET_MIGRATE_AGGREGATE_TABLE_TO_TRUE is set to 1, skipping aggregate table migration forever (for 2.0.0-dev users)"); + await Settings.set("migratedAggregateTable", true); + } + // let migrated = await Settings.get("migratedAggregateTable"); @@ -762,12 +768,13 @@ class Database { `); // Stop if stat_* tables are not empty + // SQL to empty these tables: DELETE FROM stat_minutely; DELETE FROM stat_hourly; DELETE FROM stat_daily; for (let table of [ "stat_minutely", "stat_hourly", "stat_daily" ]) { let countResult = await trx.raw(`SELECT COUNT(*) AS count FROM ${table}`); let count = countResult[0].count; if (count > 0) { log.warn("db", `Aggregate table ${table} is not empty, migration will not be started (Maybe you were using 2.0.0-dev?)`); - trx.commit(); + trx.rollback(); return; } } From 6437b9afab7edd671dcc3aa32096b8a735a9ed7c Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Wed, 9 Oct 2024 07:08:05 +0800 Subject: [PATCH 08/19] Revert apicache changes as the debug log is too crazy --- server/modules/apicache/apicache.js | 53 +++++++++++++++++++---------- 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/server/modules/apicache/apicache.js b/server/modules/apicache/apicache.js index 804fa93ed..80136d0a4 100644 --- a/server/modules/apicache/apicache.js +++ b/server/modules/apicache/apicache.js @@ -1,6 +1,5 @@ let url = require("url"); let MemoryCache = require("./memory-cache"); -const { log } = require("../../../src/util"); let t = { ms: 1, @@ -91,6 +90,24 @@ function ApiCache() { instances.push(this); this.id = instances.length; + /** + * Logs a message to the console if the `DEBUG` environment variable is set. + * @param {string} a The first argument to log. + * @param {string} b The second argument to log. + * @param {string} c The third argument to log. + * @param {string} d The fourth argument to log, and so on... (optional) + * + * Generated by Trelent + */ + function debug(a, b, c, d) { + let arr = ["\x1b[36m[apicache]\x1b[0m", a, b, c, d].filter(function (arg) { + return arg !== undefined; + }); + let debugEnv = process.env.DEBUG && process.env.DEBUG.split(",").indexOf("apicache") !== -1; + + return (globalOptions.debug || debugEnv) && console.log.apply(null, arr); + } + /** * Returns true if the given request and response should be logged. * @param {Object} request The HTTP request object. @@ -129,7 +146,7 @@ function ApiCache() { let groupName = req.apicacheGroup; if (groupName) { - log.debug("apicache", `group detected "${groupName}"`); + debug("group detected \"" + groupName + "\""); let group = (index.groups[groupName] = index.groups[groupName] || []); group.unshift(key); } @@ -195,7 +212,7 @@ function ApiCache() { redis.hset(key, "duration", duration); redis.expire(key, duration / 1000, expireCallback || function () {}); } catch (err) { - log.debug("apicache", `error in redis.hset(): ${err}`); + debug("[apicache] error in redis.hset()"); } } else { memCache.add(key, value, duration, expireCallback); @@ -303,10 +320,10 @@ function ApiCache() { // display log entry let elapsed = new Date() - req.apicacheTimer; - log.debug("apicache", `adding cache entry for "${key}" @ ${strDuration} ${logDuration(elapsed)}`); - log.debug("apicache", `_apicache.headers: ${JSON.stringify(res._apicache.headers)}`); - log.debug("apicache", `res.getHeaders(): ${JSON.stringify(getSafeHeaders(res))}`); - log.debug("apicache", `cacheObject: ${JSON.stringify(cacheObject)}`); + debug("adding cache entry for \"" + key + "\" @ " + strDuration, logDuration(elapsed)); + debug("_apicache.headers: ", res._apicache.headers); + debug("res.getHeaders(): ", getSafeHeaders(res)); + debug("cacheObject: ", cacheObject); } } @@ -385,10 +402,10 @@ function ApiCache() { let redis = globalOptions.redisClient; if (group) { - log.debug("apicache", `clearing group "${target}"`); + debug("clearing group \"" + target + "\""); group.forEach(function (key) { - log.debug("apicache", `clearing cached entry for "${key}"`); + debug("clearing cached entry for \"" + key + "\""); clearTimeout(timers[key]); delete timers[key]; if (!globalOptions.redisClient) { @@ -397,7 +414,7 @@ function ApiCache() { try { redis.del(key); } catch (err) { - log.info("apicache", "error in redis.del(\"" + key + "\")"); + console.log("[apicache] error in redis.del(\"" + key + "\")"); } } index.all = index.all.filter(doesntMatch(key)); @@ -405,7 +422,7 @@ function ApiCache() { delete index.groups[target]; } else if (target) { - log.debug("apicache", `clearing ${isAutomatic ? "expired" : "cached"} entry for "${target}"`); + debug("clearing " + (isAutomatic ? "expired" : "cached") + " entry for \"" + target + "\""); clearTimeout(timers[target]); delete timers[target]; // clear actual cached entry @@ -415,7 +432,7 @@ function ApiCache() { try { redis.del(target); } catch (err) { - log.error("apicache", "error in redis.del(\"" + target + "\")"); + console.log("[apicache] error in redis.del(\"" + target + "\")"); } } @@ -432,7 +449,7 @@ function ApiCache() { } }); } else { - log.debug("apicache", "clearing entire index"); + debug("clearing entire index"); if (!redis) { memCache.clear(); @@ -444,7 +461,7 @@ function ApiCache() { try { redis.del(key); } catch (err) { - log.error("apicache", `error in redis.del("${key}"): ${err}`); + console.log("[apicache] error in redis.del(\"" + key + "\")"); } }); } @@ -735,7 +752,7 @@ function ApiCache() { */ let cache = function (req, res, next) { function bypass() { - log.debug("apicache", "bypass detected, skipping cache."); + debug("bypass detected, skipping cache."); return next(); } @@ -788,7 +805,7 @@ function ApiCache() { // send if cache hit from memory-cache if (cached) { let elapsed = new Date() - req.apicacheTimer; - log.debug("apicache", `sending cached (memory-cache) version of ${key} ${logDuration(elapsed)}`); + debug("sending cached (memory-cache) version of", key, logDuration(elapsed)); perf.hit(key); return sendCachedResponse(req, res, cached, middlewareToggle, next, duration); @@ -800,7 +817,7 @@ function ApiCache() { redis.hgetall(key, function (err, obj) { if (!err && obj && obj.response) { let elapsed = new Date() - req.apicacheTimer; - log.debug("apicache", "sending cached (redis) version of "+ key+" "+ logDuration(elapsed)); + debug("sending cached (redis) version of", key, logDuration(elapsed)); perf.hit(key); return sendCachedResponse( @@ -856,7 +873,7 @@ function ApiCache() { } if (globalOptions.trackPerformance) { - log.debug("apicache", "WARNING: using trackPerformance flag can cause high memory usage!"); + debug("WARNING: using trackPerformance flag can cause high memory usage!"); } return this; From 344fd5250134c2d88e6fa44c76c453842660ee40 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Wed, 9 Oct 2024 07:18:55 +0800 Subject: [PATCH 09/19] Avoid Uptime Kuma getting stop if one monitor is gone wrong --- server/server.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/server.js b/server/server.js index 3579df5d2..59d636259 100644 --- a/server/server.js +++ b/server/server.js @@ -1803,7 +1803,11 @@ async function startMonitors() { } for (let monitor of list) { - await monitor.start(io); + try { + await monitor.start(io); + } catch (e) { + log.error("monitor", e); + } // Give some delays, so all monitors won't make request at the same moment when just start the server. await sleep(getRandomInt(300, 1000)); } From d7c3c40d74b63fd8eafb81c779b7f392be204c48 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Wed, 9 Oct 2024 07:19:05 +0800 Subject: [PATCH 10/19] wip --- server/database.js | 81 ++++++++++++++++++++----------------- server/uptime-calculator.js | 41 ++++++++++++++----- 2 files changed, 75 insertions(+), 47 deletions(-) diff --git a/server/database.js b/server/database.js index a82eaa813..7928bb855 100644 --- a/server/database.js +++ b/server/database.js @@ -741,6 +741,14 @@ class Database { await Settings.set("migratedAggregateTable", true); } + // Empty the aggregate table if FORCE_MIGRATE_AGGREGATE_TABLE is set to 1 + if (process.env.FORCE_MIGRATE_AGGREGATE_TABLE === "1") { + log.warn("db", "FORCE_MIGRATE_AGGREGATE_TABLE is set to 1, forcing aggregate table migration"); + await R.exec("DELETE FROM stat_minutely"); + await R.exec("DELETE FROM stat_hourly"); + await R.exec("DELETE FROM stat_daily"); + } + // let migrated = await Settings.get("migratedAggregateTable"); @@ -751,60 +759,61 @@ class Database { log.info("db", "Migrating Aggregate Table"); - // Migrate heartbeat to stat_minutely, using knex transaction - const trx = await R.knex.transaction(); + log.info("db", "Getting list of unique dates and monitors"); // Get a list of unique dates from the heartbeat table, using raw sql - let dates = await trx.raw(` + let dates = await R.getAll(` SELECT DISTINCT DATE(time) AS date FROM heartbeat ORDER BY date ASC `); // Get a list of unique monitors from the heartbeat table, using raw sql - let monitors = await trx.raw(` + let monitors = await R.getAll(` SELECT DISTINCT monitor_id FROM heartbeat + ORDER BY monitor_id ASC `); - // Stop if stat_* tables are not empty - // SQL to empty these tables: DELETE FROM stat_minutely; DELETE FROM stat_hourly; DELETE FROM stat_daily; - for (let table of [ "stat_minutely", "stat_hourly", "stat_daily" ]) { - let countResult = await trx.raw(`SELECT COUNT(*) AS count FROM ${table}`); - let count = countResult[0].count; - if (count > 0) { - log.warn("db", `Aggregate table ${table} is not empty, migration will not be started (Maybe you were using 2.0.0-dev?)`); - trx.rollback(); - return; - } - } - console.log("Dates", dates); console.log("Monitors", monitors); - for (let monitor of monitors) { - for (let date of dates) { - log.info("db", `Migrating monitor ${monitor.monitor_id} on date ${date.date}`); - - // New Uptime Calculator - let calculator = new UptimeCalculator(); - - // TODO: Pass transaction to the calculator - // calculator.setTransaction(trx); - - // Get all the heartbeats for this monitor and date - let heartbeats = await trx("heartbeat") - .where("monitor_id", monitor.monitor_id) - .whereRaw("DATE(time) = ?", [ date.date ]) - .orderBy("time", "asc"); - - for (let heartbeat of heartbeats) { - calculator.update(heartbeat.status, heartbeat.ping, dayjs(heartbeat.time)); - } + // Stop if stat_* tables are not empty + for (let table of [ "stat_minutely", "stat_hourly", "stat_daily" ]) { + let countResult = await R.getRow(`SELECT COUNT(*) AS count FROM ${table}`); + let count = countResult.count; + if (count > 0) { + log.warn("db", `Aggregate table ${table} is not empty, migration will not be started (Maybe you were using 2.0.0-dev?)`); + return; } } - trx.commit(); + for (let monitor of monitors) { + for (let date of dates) { + + // New Uptime Calculator + let calculator = new UptimeCalculator(); + calculator.monitorID = monitor.monitor_id; + calculator.setMigrationMode(true); + + // Get all the heartbeats for this monitor and date + let heartbeats = await R.getAll(` + SELECT status, ping, time + FROM heartbeat + WHERE monitor_id = ? + AND DATE(time) = ? + ORDER BY time ASC + `, [ monitor.monitor_id, date.date ]); + + if (heartbeats.length > 0) { + log.info("db", `Migrating monitor ${monitor.monitor_id} on date ${date.date}`); + } + + for (let heartbeat of heartbeats) { + await calculator.update(heartbeat.status, parseFloat(heartbeat.ping), dayjs(heartbeat.time)); + } + } + } //await Settings.set("migratedAggregateTable", true); } diff --git a/server/uptime-calculator.js b/server/uptime-calculator.js index e9ade777e..ea2118b51 100644 --- a/server/uptime-calculator.js +++ b/server/uptime-calculator.js @@ -12,7 +12,6 @@ class UptimeCalculator { * @private * @type {{string:UptimeCalculator}} */ - static list = {}; /** @@ -55,6 +54,12 @@ class UptimeCalculator { lastHourlyStatBean = null; lastMinutelyStatBean = null; + /** + * For migration purposes. + * @type {boolean} + */ + migrationMode = false; + /** * Get the uptime calculator for a monitor * Initializes and returns the monitor if it does not exist @@ -194,6 +199,10 @@ class UptimeCalculator { * @throws {Error} Invalid status */ async update(status, ping = 0, date) { + if (!this.monitorID) { + throw new Error("Monitor ID is required"); + } + if (!date) { date = this.getCurrentDate(); } @@ -330,17 +339,19 @@ class UptimeCalculator { } await R.store(minutelyStatBean); - // Remove the old data - log.debug("uptime-calc", "Remove old data"); - await R.exec("DELETE FROM stat_minutely WHERE monitor_id = ? AND timestamp < ?", [ - this.monitorID, - this.getMinutelyKey(date.subtract(24, "hour")), - ]); + if (!this.migrationMode) { + // Remove the old data + log.debug("uptime-calc", "Remove old data"); + await R.exec("DELETE FROM stat_minutely WHERE monitor_id = ? AND timestamp < ?", [ + this.monitorID, + this.getMinutelyKey(date.subtract(24, "hour")), + ]); - await R.exec("DELETE FROM stat_hourly WHERE monitor_id = ? AND timestamp < ?", [ - this.monitorID, - this.getHourlyKey(date.subtract(30, "day")), - ]); + await R.exec("DELETE FROM stat_hourly WHERE monitor_id = ? AND timestamp < ?", [ + this.monitorID, + this.getHourlyKey(date.subtract(30, "day")), + ]); + } return date; } @@ -815,6 +826,14 @@ class UptimeCalculator { return dayjs.utc(); } + /** + * For migration purposes. + * @param {boolean} value Migration mode on/off + * @returns {void} + */ + setMigrationMode(value) { + this.migrationMode = value; + } } class UptimeDataResult { From 03e507a4e1d819268755789096e365471b4ba2f4 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Wed, 9 Oct 2024 07:46:41 +0800 Subject: [PATCH 11/19] wip --- server/database.js | 1 + 1 file changed, 1 insertion(+) diff --git a/server/database.js b/server/database.js index 5bcd79887..c8a33ac20 100644 --- a/server/database.js +++ b/server/database.js @@ -6,6 +6,7 @@ const knex = require("knex"); const path = require("path"); const { EmbeddedMariaDB } = require("./embedded-mariadb"); const mysql = require("mysql2/promise"); +const { Settings } = require("./settings"); const { UptimeCalculator } = require("./uptime-calculator"); const dayjs = require("dayjs"); From 0f646e634e5684b8c09df3dd2d24eaa9ad18d25e Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Wed, 9 Oct 2024 13:12:41 +0800 Subject: [PATCH 12/19] wip --- server/database.js | 7 +++---- server/uptime-calculator.js | 1 + 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/database.js b/server/database.js index c8a33ac20..ea0db663c 100644 --- a/server/database.js +++ b/server/database.js @@ -776,9 +776,6 @@ class Database { ORDER BY monitor_id ASC `); - console.log("Dates", dates); - console.log("Monitors", monitors); - // Stop if stat_* tables are not empty for (let table of [ "stat_minutely", "stat_hourly", "stat_daily" ]) { let countResult = await R.getRow(`SELECT COUNT(*) AS count FROM ${table}`); @@ -790,6 +787,8 @@ class Database { } for (let monitor of monitors) { + + // TODO: Get the date list for each monitor for (let date of dates) { // New Uptime Calculator @@ -816,7 +815,7 @@ class Database { } } - //await Settings.set("migratedAggregateTable", true); + await Settings.set("migratedAggregateTable", true); } } diff --git a/server/uptime-calculator.js b/server/uptime-calculator.js index ea2118b51..43f6b5a56 100644 --- a/server/uptime-calculator.js +++ b/server/uptime-calculator.js @@ -339,6 +339,7 @@ class UptimeCalculator { } await R.store(minutelyStatBean); + // TODO: it seems that it is also necessary to remove the old data in the migration mode if (!this.migrationMode) { // Remove the old data log.debug("uptime-calc", "Remove old data"); From 19a9735234696be0ef9cd821cf358ac853e33922 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Wed, 9 Oct 2024 13:15:29 +0800 Subject: [PATCH 13/19] Fix merging issue --- server/jobs/clear-old-data.js | 1 + 1 file changed, 1 insertion(+) diff --git a/server/jobs/clear-old-data.js b/server/jobs/clear-old-data.js index ada7f8882..eedc628aa 100644 --- a/server/jobs/clear-old-data.js +++ b/server/jobs/clear-old-data.js @@ -2,6 +2,7 @@ const { R } = require("redbean-node"); const { log } = require("../../src/util"); const { setSetting, setting } = require("../util-server"); const Database = require("../database"); +const Settings = require("../models/settings"); const DEFAULT_KEEP_PERIOD = 180; From c86b12d5d2e1e225a444a76c09def2948db2806f Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Wed, 9 Oct 2024 13:16:28 +0800 Subject: [PATCH 14/19] Fix merging issue --- server/jobs/clear-old-data.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/jobs/clear-old-data.js b/server/jobs/clear-old-data.js index eedc628aa..956be10e7 100644 --- a/server/jobs/clear-old-data.js +++ b/server/jobs/clear-old-data.js @@ -2,7 +2,7 @@ const { R } = require("redbean-node"); const { log } = require("../../src/util"); const { setSetting, setting } = require("../util-server"); const Database = require("../database"); -const Settings = require("../models/settings"); +const { Settings } = require("../settings"); const DEFAULT_KEEP_PERIOD = 180; From 776f4f2d5ffe9f04e399e5eadb79544971cc030a Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Fri, 11 Oct 2024 17:10:29 +0800 Subject: [PATCH 15/19] wip --- server/uptime-calculator.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/server/uptime-calculator.js b/server/uptime-calculator.js index 43f6b5a56..5b27fe6fb 100644 --- a/server/uptime-calculator.js +++ b/server/uptime-calculator.js @@ -199,10 +199,6 @@ class UptimeCalculator { * @throws {Error} Invalid status */ async update(status, ping = 0, date) { - if (!this.monitorID) { - throw new Error("Monitor ID is required"); - } - if (!date) { date = this.getCurrentDate(); } From 4632030a5e922cba6cf37063bf3a8627e4e29df7 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Fri, 11 Oct 2024 18:06:59 +0800 Subject: [PATCH 16/19] wip --- extra/reset-migrate-aggregate-table-status.js | 24 +++++++ package.json | 3 +- server/database.js | 71 ++++++++++++------- server/server.js | 10 +-- 4 files changed, 78 insertions(+), 30 deletions(-) create mode 100644 extra/reset-migrate-aggregate-table-status.js diff --git a/extra/reset-migrate-aggregate-table-status.js b/extra/reset-migrate-aggregate-table-status.js new file mode 100644 index 000000000..e6c51fbd0 --- /dev/null +++ b/extra/reset-migrate-aggregate-table-status.js @@ -0,0 +1,24 @@ +const { R } = require("redbean-node"); +const Database = require("../server/database"); +const args = require("args-parser")(process.argv); +const { Settings } = require("../server/settings"); + +const main = async () => { + console.log("Connecting the database"); + Database.initDataDir(args); + await Database.connect(false, false, true); + + console.log("Deleting all data from aggregate tables"); + await R.exec("DELETE FROM stat_minutely"); + await R.exec("DELETE FROM stat_hourly"); + await R.exec("DELETE FROM stat_daily"); + + console.log("Resetting the aggregate table state"); + await Settings.set("migrateAggregateTableState", ""); + + await Database.close(); + console.log("Done"); +}; + +main(); + diff --git a/package.json b/package.json index 0d51984c7..8ba835964 100644 --- a/package.json +++ b/package.json @@ -68,7 +68,8 @@ "sort-contributors": "node extra/sort-contributors.js", "quick-run-nightly": "docker run --rm --env NODE_ENV=development -p 3001:3001 louislam/uptime-kuma:nightly2", "start-dev-container": "cd docker && docker-compose -f docker-compose-dev.yml up --force-recreate", - "rebase-pr-to-1.23.X": "node extra/rebase-pr.js 1.23.X" + "rebase-pr-to-1.23.X": "node extra/rebase-pr.js 1.23.X", + "reset-migrate-aggregate-table-status": "node extra/reset-migrate-aggregate-table-status.js" }, "dependencies": { "@grpc/grpc-js": "~1.8.22", diff --git a/server/database.js b/server/database.js index ea0db663c..4b2573150 100644 --- a/server/database.js +++ b/server/database.js @@ -739,35 +739,26 @@ class Database { // Add a setting for 2.0.0-dev users to skip this migration if (process.env.SET_MIGRATE_AGGREGATE_TABLE_TO_TRUE === "1") { log.warn("db", "SET_MIGRATE_AGGREGATE_TABLE_TO_TRUE is set to 1, skipping aggregate table migration forever (for 2.0.0-dev users)"); - await Settings.set("migratedAggregateTable", true); + await Settings.set("migrateAggregateTableState", "migrated"); } - // Empty the aggregate table if FORCE_MIGRATE_AGGREGATE_TABLE is set to 1 - if (process.env.FORCE_MIGRATE_AGGREGATE_TABLE === "1") { - log.warn("db", "FORCE_MIGRATE_AGGREGATE_TABLE is set to 1, forcing aggregate table migration"); - await R.exec("DELETE FROM stat_minutely"); - await R.exec("DELETE FROM stat_hourly"); - await R.exec("DELETE FROM stat_daily"); - } + let migrateState = await Settings.get("migrateAggregateTableState"); - // - let migrated = await Settings.get("migratedAggregateTable"); - - if (migrated) { - log.debug("db", "Migrated, skip migration"); + // Skip if already migrated + // If it is migrating, it possibly means the migration was interrupted, or the migration is in progress + if (migrateState === "migrated") { + log.debug("db", "Migrated aggregate table already, skip"); return; + } else if (migrateState === "migrating") { + log.warn("db", "Aggregate table migration is already in progress, or it was interrupted"); + throw new Error("Aggregate table migration is already in progress"); } + await Settings.set("migrateAggregateTableState", "migrating"); + log.info("db", "Migrating Aggregate Table"); - log.info("db", "Getting list of unique dates and monitors"); - - // Get a list of unique dates from the heartbeat table, using raw sql - let dates = await R.getAll(` - SELECT DISTINCT DATE(time) AS date - FROM heartbeat - ORDER BY date ASC - `); + log.info("db", "Getting list of unique monitors"); // Get a list of unique monitors from the heartbeat table, using raw sql let monitors = await R.getAll(` @@ -786,11 +777,23 @@ class Database { } } + let progressPercent = 0; + let part = 100 / monitors.length; + let i = 1; for (let monitor of monitors) { - // TODO: Get the date list for each monitor - for (let date of dates) { + // TODO: Get two or three days at the same to speed up??? + // Get a list of unique dates from the heartbeat table, using raw sql + let dates = await R.getAll(` + SELECT DISTINCT DATE(time) AS date + FROM heartbeat + WHERE monitor_id = ? + ORDER BY date ASC + `, [ + monitor.monitor_id + ]); + for (let date of dates) { // New Uptime Calculator let calculator = new UptimeCalculator(); calculator.monitorID = monitor.monitor_id; @@ -806,16 +809,34 @@ class Database { `, [ monitor.monitor_id, date.date ]); if (heartbeats.length > 0) { - log.info("db", `Migrating monitor ${monitor.monitor_id} on date ${date.date}`); + log.info("db", `[DON'T STOP] Migrating monitor data ${monitor.monitor_id} - ${date.date} [${progressPercent.toFixed(2)}%][${i}/${monitors.length}]`); } for (let heartbeat of heartbeats) { await calculator.update(heartbeat.status, parseFloat(heartbeat.ping), dayjs(heartbeat.time)); } + + progressPercent += (Math.round(part / dates.length * 100) / 100); + + // Lazy to fix the floating point issue, it is acceptable since it is just a progress bar + if (progressPercent > 100) { + progressPercent = 100; + } } + + i++; } - await Settings.set("migratedAggregateTable", true); + // TODO: Remove all non-important heartbeats from heartbeat table + log.info("db", "[DON'T STOP] Deleting all data from heartbeat table"); + + await Settings.set("migrateAggregateTableState", "migrated"); + + if (monitors.length > 0) { + log.info("db", "Aggregate Table Migration Completed"); + } else { + log.info("db", "No data to migrate"); + } } } diff --git a/server/server.js b/server/server.js index 669c8dc54..93f74f349 100644 --- a/server/server.js +++ b/server/server.js @@ -1599,18 +1599,20 @@ let needSetup = false; await server.start(); - server.httpServer.listen(port, hostname, () => { + server.httpServer.listen(port, hostname, async () => { if (hostname) { log.info("server", `Listening on ${hostname}:${port}`); } else { log.info("server", `Listening on ${port}`); } - startMonitors(); + await startMonitors(); + + // Put this here. Start background jobs after the db and server is ready to prevent clear up during db migration. + await initBackgroundJobs(); + checkVersion.startInterval(); }); - await initBackgroundJobs(); - // Start cloudflared at the end if configured await cloudflaredAutoStart(cloudflaredToken); From fe91ffcc9dadedca0f6eb002c86734e3cc631b65 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Sun, 13 Oct 2024 00:51:55 +0800 Subject: [PATCH 17/19] wip --- server/jobs/clear-old-data.js | 3 ++ server/uptime-calculator.js | 55 ++++++++++++++++++----------------- 2 files changed, 32 insertions(+), 26 deletions(-) diff --git a/server/jobs/clear-old-data.js b/server/jobs/clear-old-data.js index 956be10e7..fcdeac187 100644 --- a/server/jobs/clear-old-data.js +++ b/server/jobs/clear-old-data.js @@ -13,6 +13,9 @@ const DEFAULT_KEEP_PERIOD = 180; const clearOldData = async () => { + // TODO: Temporary disable for testing + return; + /* * TODO: * Since we have aggregated table now, we don't need so much data in heartbeat table. diff --git a/server/uptime-calculator.js b/server/uptime-calculator.js index 5b27fe6fb..916f4c4cc 100644 --- a/server/uptime-calculator.js +++ b/server/uptime-calculator.js @@ -305,37 +305,40 @@ class UptimeCalculator { } await R.store(dailyStatBean); - let hourlyStatBean = await this.getHourlyStatBean(hourlyKey); - hourlyStatBean.up = hourlyData.up; - hourlyStatBean.down = hourlyData.down; - hourlyStatBean.ping = hourlyData.avgPing; - hourlyStatBean.pingMin = hourlyData.minPing; - hourlyStatBean.pingMax = hourlyData.maxPing; - { - // eslint-disable-next-line no-unused-vars - const { up, down, avgPing, minPing, maxPing, timestamp, ...extras } = hourlyData; - if (Object.keys(extras).length > 0) { - hourlyStatBean.extras = JSON.stringify(extras); + // TODO: For migration mode, we don't need to store old hourly and minutely data, but we need 24-hour's minutely data and 30-day's hourly data + if (false) { + let hourlyStatBean = await this.getHourlyStatBean(hourlyKey); + hourlyStatBean.up = hourlyData.up; + hourlyStatBean.down = hourlyData.down; + hourlyStatBean.ping = hourlyData.avgPing; + hourlyStatBean.pingMin = hourlyData.minPing; + hourlyStatBean.pingMax = hourlyData.maxPing; + { + // eslint-disable-next-line no-unused-vars + const { up, down, avgPing, minPing, maxPing, timestamp, ...extras } = hourlyData; + if (Object.keys(extras).length > 0) { + hourlyStatBean.extras = JSON.stringify(extras); + } } - } - await R.store(hourlyStatBean); + await R.store(hourlyStatBean); - let minutelyStatBean = await this.getMinutelyStatBean(divisionKey); - minutelyStatBean.up = minutelyData.up; - minutelyStatBean.down = minutelyData.down; - minutelyStatBean.ping = minutelyData.avgPing; - minutelyStatBean.pingMin = minutelyData.minPing; - minutelyStatBean.pingMax = minutelyData.maxPing; - { - // eslint-disable-next-line no-unused-vars - const { up, down, avgPing, minPing, maxPing, timestamp, ...extras } = minutelyData; - if (Object.keys(extras).length > 0) { - minutelyStatBean.extras = JSON.stringify(extras); + let minutelyStatBean = await this.getMinutelyStatBean(divisionKey); + minutelyStatBean.up = minutelyData.up; + minutelyStatBean.down = minutelyData.down; + minutelyStatBean.ping = minutelyData.avgPing; + minutelyStatBean.pingMin = minutelyData.minPing; + minutelyStatBean.pingMax = minutelyData.maxPing; + { + // eslint-disable-next-line no-unused-vars + const { up, down, avgPing, minPing, maxPing, timestamp, ...extras } = minutelyData; + if (Object.keys(extras).length > 0) { + minutelyStatBean.extras = JSON.stringify(extras); + } } + await R.store(minutelyStatBean); } - await R.store(minutelyStatBean); - // TODO: it seems that it is also necessary to remove the old data in the migration mode + // No need to remove old data in migration mode if (!this.migrationMode) { // Remove the old data log.debug("uptime-calc", "Remove old data"); From 93cc21271f248d41a1fd9c92a9052a6f5a2e9fad Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Tue, 15 Oct 2024 00:40:46 +0800 Subject: [PATCH 18/19] wip --- ...ble-status.js => reset-migrate-aggregate-table-state.js} | 0 package.json | 2 +- server/database.js | 6 +++++- 3 files changed, 6 insertions(+), 2 deletions(-) rename extra/{reset-migrate-aggregate-table-status.js => reset-migrate-aggregate-table-state.js} (100%) diff --git a/extra/reset-migrate-aggregate-table-status.js b/extra/reset-migrate-aggregate-table-state.js similarity index 100% rename from extra/reset-migrate-aggregate-table-status.js rename to extra/reset-migrate-aggregate-table-state.js diff --git a/package.json b/package.json index 8ba835964..1e2982e18 100644 --- a/package.json +++ b/package.json @@ -69,7 +69,7 @@ "quick-run-nightly": "docker run --rm --env NODE_ENV=development -p 3001:3001 louislam/uptime-kuma:nightly2", "start-dev-container": "cd docker && docker-compose -f docker-compose-dev.yml up --force-recreate", "rebase-pr-to-1.23.X": "node extra/rebase-pr.js 1.23.X", - "reset-migrate-aggregate-table-status": "node extra/reset-migrate-aggregate-table-status.js" + "reset-migrate-aggregate-table-state": "node extra/reset-migrate-aggregate-table-state.js" }, "dependencies": { "@grpc/grpc-js": "~1.8.22", diff --git a/server/database.js b/server/database.js index 4b2573150..8ff886045 100644 --- a/server/database.js +++ b/server/database.js @@ -729,8 +729,12 @@ class Database { } /** - * TODO: Migrate the old data in the heartbeat table to the new format (stat_daily, stat_hourly, stat_minutely) + * Migrate the old data in the heartbeat table to the new format (stat_daily, stat_hourly, stat_minutely) * It should be run once while upgrading V1 to V2 + * + * Normally, it should be in transaction, but UptimeCalculator wasn't designed to be in transaction before that. + * I don't want to heavily modify the UptimeCalculator, so it is not in transaction. + * Run `npm run reset-migrate-aggregate-table-state` to reset, in case the migration is interrupted. * @returns {Promise} */ static async migrateAggregateTable() { From 5e55215c9c32129835a3107107fae618a46abdca Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Wed, 16 Oct 2024 05:12:06 +0800 Subject: [PATCH 19/19] wip --- server/database.js | 2 -- server/uptime-calculator.js | 19 +++++++++++++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/server/database.js b/server/database.js index 8ff886045..ce4e04e96 100644 --- a/server/database.js +++ b/server/database.js @@ -785,8 +785,6 @@ class Database { let part = 100 / monitors.length; let i = 1; for (let monitor of monitors) { - - // TODO: Get two or three days at the same to speed up??? // Get a list of unique dates from the heartbeat table, using raw sql let dates = await R.getAll(` SELECT DISTINCT DATE(time) AS date diff --git a/server/uptime-calculator.js b/server/uptime-calculator.js index 916f4c4cc..52a34fc29 100644 --- a/server/uptime-calculator.js +++ b/server/uptime-calculator.js @@ -60,6 +60,9 @@ class UptimeCalculator { */ migrationMode = false; + statMinutelyKeepHour = 24; + statHourlyKeepDay = 30; + /** * Get the uptime calculator for a monitor * Initializes and returns the monitor if it does not exist @@ -305,8 +308,11 @@ class UptimeCalculator { } await R.store(dailyStatBean); - // TODO: For migration mode, we don't need to store old hourly and minutely data, but we need 24-hour's minutely data and 30-day's hourly data - if (false) { + let currentDate = this.getCurrentDate(); + + // For migration mode, we don't need to store old hourly and minutely data, but we need 30-day's hourly data + // Run anyway for non-migration mode + if (!this.migrationMode || date.isAfter(currentDate.subtract(this.statHourlyKeepDay, "day"))) { let hourlyStatBean = await this.getHourlyStatBean(hourlyKey); hourlyStatBean.up = hourlyData.up; hourlyStatBean.down = hourlyData.down; @@ -321,7 +327,11 @@ class UptimeCalculator { } } await R.store(hourlyStatBean); + } + // For migration mode, we don't need to store old hourly and minutely data, but we need 24-hour's minutely data + // Run anyway for non-migration mode + if (!this.migrationMode || date.isAfter(currentDate.subtract(this.statMinutelyKeepHour, "hour"))) { let minutelyStatBean = await this.getMinutelyStatBean(divisionKey); minutelyStatBean.up = minutelyData.up; minutelyStatBean.down = minutelyData.down; @@ -341,15 +351,16 @@ class UptimeCalculator { // No need to remove old data in migration mode if (!this.migrationMode) { // Remove the old data + // TODO: Improvement: Convert it to a job? log.debug("uptime-calc", "Remove old data"); await R.exec("DELETE FROM stat_minutely WHERE monitor_id = ? AND timestamp < ?", [ this.monitorID, - this.getMinutelyKey(date.subtract(24, "hour")), + this.getMinutelyKey(currentDate.subtract(this.statMinutelyKeepHour, "hour")), ]); await R.exec("DELETE FROM stat_hourly WHERE monitor_id = ? AND timestamp < ?", [ this.monitorID, - this.getHourlyKey(date.subtract(30, "day")), + this.getHourlyKey(currentDate.subtract(this.statHourlyKeepDay, "day")), ]); }