diff --git a/server/database.js b/server/database.js index ce4e04e96..16cdecc5c 100644 --- a/server/database.js +++ b/server/database.js @@ -829,8 +829,7 @@ class Database { i++; } - // TODO: Remove all non-important heartbeats from heartbeat table - log.info("db", "[DON'T STOP] Deleting all data from heartbeat table"); + await Database.clearHeartbeatData(true); await Settings.set("migrateAggregateTableState", "migrated"); @@ -841,6 +840,40 @@ class Database { } } + /** + * Remove all non-important heartbeats from heartbeat table, keep last 24-hour or {KEEP_LAST_ROWS} rows for each monitor + * @param {boolean} detailedLog + * @returns {Promise} + */ + static async clearHeartbeatData(detailedLog = false) { + let monitors = await R.getAll("SELECT id FROM monitor"); + const sqlHourOffset = Database.sqlHourOffset(); + + for (let monitor of monitors) { + if (detailedLog) { + log.info("db", "Deleting non-important heartbeats for monitor " + monitor.id); + } + await R.exec(` + DELETE FROM heartbeat + WHERE monitor_id = ? + AND important = 0 + AND time < ${sqlHourOffset} + AND id NOT IN ( + SELECT id + FROM heartbeat + WHERE monitor_id = ? + ORDER BY time DESC + LIMIT ? + ) + `, [ + monitor.id, + -24, + monitor.id, + 100, + ]); + } + } + } module.exports = Database; diff --git a/server/jobs/clear-old-data.js b/server/jobs/clear-old-data.js index fcdeac187..df5a943a8 100644 --- a/server/jobs/clear-old-data.js +++ b/server/jobs/clear-old-data.js @@ -1,35 +1,19 @@ const { R } = require("redbean-node"); const { log } = require("../../src/util"); -const { setSetting, setting } = require("../util-server"); +const { setSetting } = require("../util-server"); const Database = require("../database"); const { Settings } = require("../settings"); +const dayjs = require("dayjs"); -const DEFAULT_KEEP_PERIOD = 180; +const DEFAULT_KEEP_PERIOD = 365; /** - * Clears old data from the heartbeat table of the database. + * Clears old data from the heartbeat table and the stat_daily of the database. * @returns {Promise} A promise that resolves when the data has been cleared. */ - const clearOldData = async () => { - - // TODO: Temporary disable for testing - return; - - /* - * TODO: - * Since we have aggregated table now, we don't need so much data in heartbeat table. - * But we still need to keep the important rows, because they contain the message. - * - * In the heartbeat table: - * - important rows: keep according to the setting (keepDataPeriodDays) (default 180 days) - * - not important rows: keep 2 days - * - * stat_* tables: - * - keep according to the setting (keepDataPeriodDays) (default 180 days) - */ - - let period = await setting("keepDataPeriodDays"); + await Database.clearHeartbeatData(); + let period = await Settings.get("keepDataPeriodDays"); // Set Default Period if (period == null) { @@ -50,16 +34,21 @@ const clearOldData = async () => { if (parsedPeriod < 1) { log.info("clearOldData", `Data deletion has been disabled as period is less than 1. Period is ${parsedPeriod} days.`); } else { - log.debug("clearOldData", `Clearing Data older than ${parsedPeriod} days...`); - const sqlHourOffset = Database.sqlHourOffset(); try { - await R.exec( - "DELETE FROM heartbeat WHERE time < " + sqlHourOffset, - [ parsedPeriod * -24 ] - ); + // Heartbeat + await R.exec("DELETE FROM heartbeat WHERE time < " + sqlHourOffset, [ + parsedPeriod * -24, + ]); + + let timestamp = dayjs().subtract(parsedPeriod, "day").utc().startOf("day").unix(); + + // stat_daily + await R.exec("DELETE FROM stat_daily WHERE timestamp < ? ", [ + timestamp, + ]); if (Database.dbConfig.type === "sqlite") { await R.exec("PRAGMA optimize;"); @@ -68,6 +57,8 @@ const clearOldData = async () => { log.error("clearOldData", `Failed to clear old data: ${e.message}`); } } + + log.debug("clearOldData", "Data cleared."); }; module.exports = { diff --git a/server/uptime-calculator.js b/server/uptime-calculator.js index 52a34fc29..71d1d458c 100644 --- a/server/uptime-calculator.js +++ b/server/uptime-calculator.js @@ -209,7 +209,7 @@ class UptimeCalculator { let flatStatus = this.flatStatus(status); if (flatStatus === DOWN && ping > 0) { - log.warn("uptime-calc", "The ping is not effective when the status is DOWN"); + log.debug("uptime-calc", "The ping is not effective when the status is DOWN"); } let divisionKey = this.getMinutelyKey(date);