From 4632030a5e922cba6cf37063bf3a8627e4e29df7 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Fri, 11 Oct 2024 18:06:59 +0800 Subject: [PATCH] wip --- extra/reset-migrate-aggregate-table-status.js | 24 +++++++ package.json | 3 +- server/database.js | 71 ++++++++++++------- server/server.js | 10 +-- 4 files changed, 78 insertions(+), 30 deletions(-) create mode 100644 extra/reset-migrate-aggregate-table-status.js diff --git a/extra/reset-migrate-aggregate-table-status.js b/extra/reset-migrate-aggregate-table-status.js new file mode 100644 index 000000000..e6c51fbd0 --- /dev/null +++ b/extra/reset-migrate-aggregate-table-status.js @@ -0,0 +1,24 @@ +const { R } = require("redbean-node"); +const Database = require("../server/database"); +const args = require("args-parser")(process.argv); +const { Settings } = require("../server/settings"); + +const main = async () => { + console.log("Connecting the database"); + Database.initDataDir(args); + await Database.connect(false, false, true); + + console.log("Deleting all data from aggregate tables"); + await R.exec("DELETE FROM stat_minutely"); + await R.exec("DELETE FROM stat_hourly"); + await R.exec("DELETE FROM stat_daily"); + + console.log("Resetting the aggregate table state"); + await Settings.set("migrateAggregateTableState", ""); + + await Database.close(); + console.log("Done"); +}; + +main(); + diff --git a/package.json b/package.json index 0d51984c7..8ba835964 100644 --- a/package.json +++ b/package.json @@ -68,7 +68,8 @@ "sort-contributors": "node extra/sort-contributors.js", "quick-run-nightly": "docker run --rm --env NODE_ENV=development -p 3001:3001 louislam/uptime-kuma:nightly2", "start-dev-container": "cd docker && docker-compose -f docker-compose-dev.yml up --force-recreate", - "rebase-pr-to-1.23.X": "node extra/rebase-pr.js 1.23.X" + "rebase-pr-to-1.23.X": "node extra/rebase-pr.js 1.23.X", + "reset-migrate-aggregate-table-status": "node extra/reset-migrate-aggregate-table-status.js" }, "dependencies": { "@grpc/grpc-js": "~1.8.22", diff --git a/server/database.js b/server/database.js index ea0db663c..4b2573150 100644 --- a/server/database.js +++ b/server/database.js @@ -739,35 +739,26 @@ class Database { // Add a setting for 2.0.0-dev users to skip this migration if (process.env.SET_MIGRATE_AGGREGATE_TABLE_TO_TRUE === "1") { log.warn("db", "SET_MIGRATE_AGGREGATE_TABLE_TO_TRUE is set to 1, skipping aggregate table migration forever (for 2.0.0-dev users)"); - await Settings.set("migratedAggregateTable", true); + await Settings.set("migrateAggregateTableState", "migrated"); } - // Empty the aggregate table if FORCE_MIGRATE_AGGREGATE_TABLE is set to 1 - if (process.env.FORCE_MIGRATE_AGGREGATE_TABLE === "1") { - log.warn("db", "FORCE_MIGRATE_AGGREGATE_TABLE is set to 1, forcing aggregate table migration"); - await R.exec("DELETE FROM stat_minutely"); - await R.exec("DELETE FROM stat_hourly"); - await R.exec("DELETE FROM stat_daily"); - } + let migrateState = await Settings.get("migrateAggregateTableState"); - // - let migrated = await Settings.get("migratedAggregateTable"); - - if (migrated) { - log.debug("db", "Migrated, skip migration"); + // Skip if already migrated + // If it is migrating, it possibly means the migration was interrupted, or the migration is in progress + if (migrateState === "migrated") { + log.debug("db", "Migrated aggregate table already, skip"); return; + } else if (migrateState === "migrating") { + log.warn("db", "Aggregate table migration is already in progress, or it was interrupted"); + throw new Error("Aggregate table migration is already in progress"); } + await Settings.set("migrateAggregateTableState", "migrating"); + log.info("db", "Migrating Aggregate Table"); - log.info("db", "Getting list of unique dates and monitors"); - - // Get a list of unique dates from the heartbeat table, using raw sql - let dates = await R.getAll(` - SELECT DISTINCT DATE(time) AS date - FROM heartbeat - ORDER BY date ASC - `); + log.info("db", "Getting list of unique monitors"); // Get a list of unique monitors from the heartbeat table, using raw sql let monitors = await R.getAll(` @@ -786,11 +777,23 @@ class Database { } } + let progressPercent = 0; + let part = 100 / monitors.length; + let i = 1; for (let monitor of monitors) { - // TODO: Get the date list for each monitor - for (let date of dates) { + // TODO: Get two or three days at the same to speed up??? + // Get a list of unique dates from the heartbeat table, using raw sql + let dates = await R.getAll(` + SELECT DISTINCT DATE(time) AS date + FROM heartbeat + WHERE monitor_id = ? + ORDER BY date ASC + `, [ + monitor.monitor_id + ]); + for (let date of dates) { // New Uptime Calculator let calculator = new UptimeCalculator(); calculator.monitorID = monitor.monitor_id; @@ -806,16 +809,34 @@ class Database { `, [ monitor.monitor_id, date.date ]); if (heartbeats.length > 0) { - log.info("db", `Migrating monitor ${monitor.monitor_id} on date ${date.date}`); + log.info("db", `[DON'T STOP] Migrating monitor data ${monitor.monitor_id} - ${date.date} [${progressPercent.toFixed(2)}%][${i}/${monitors.length}]`); } for (let heartbeat of heartbeats) { await calculator.update(heartbeat.status, parseFloat(heartbeat.ping), dayjs(heartbeat.time)); } + + progressPercent += (Math.round(part / dates.length * 100) / 100); + + // Lazy to fix the floating point issue, it is acceptable since it is just a progress bar + if (progressPercent > 100) { + progressPercent = 100; + } } + + i++; } - await Settings.set("migratedAggregateTable", true); + // TODO: Remove all non-important heartbeats from heartbeat table + log.info("db", "[DON'T STOP] Deleting all data from heartbeat table"); + + await Settings.set("migrateAggregateTableState", "migrated"); + + if (monitors.length > 0) { + log.info("db", "Aggregate Table Migration Completed"); + } else { + log.info("db", "No data to migrate"); + } } } diff --git a/server/server.js b/server/server.js index 669c8dc54..93f74f349 100644 --- a/server/server.js +++ b/server/server.js @@ -1599,18 +1599,20 @@ let needSetup = false; await server.start(); - server.httpServer.listen(port, hostname, () => { + server.httpServer.listen(port, hostname, async () => { if (hostname) { log.info("server", `Listening on ${hostname}:${port}`); } else { log.info("server", `Listening on ${port}`); } - startMonitors(); + await startMonitors(); + + // Put this here. Start background jobs after the db and server is ready to prevent clear up during db migration. + await initBackgroundJobs(); + checkVersion.startInterval(); }); - await initBackgroundJobs(); - // Start cloudflared at the end if configured await cloudflaredAutoStart(cloudflaredToken);