mirror of https://github.com/Budibase/budibase.git
Rory Powell
4 years ago
committed by
GitHub
GitHub
23 changed files with 462 additions and 103 deletions
@ -1,30 +0,0 @@ |
|||
const { |
|||
MIGRATIONS, |
|||
MIGRATION_DBS, |
|||
migrateIfRequired, |
|||
} = require("@budibase/backend-core/migrations") |
|||
const { getGlobalDB } = require("@budibase/backend-core/tenancy") |
|||
const { getAllApps } = require("@budibase/backend-core/db") |
|||
const CouchDB = require("../db") |
|||
const { getUsageQuotaDoc } = require("../utilities/usageQuota") |
|||
|
|||
exports.runIfRequired = async () => { |
|||
await migrateIfRequired( |
|||
MIGRATION_DBS.GLOBAL_DB, |
|||
MIGRATIONS.SYNC_APP_AND_RESET_ROWS_QUOTAS, |
|||
async () => { |
|||
const db = getGlobalDB() |
|||
const usageDoc = await getUsageQuotaDoc(db) |
|||
|
|||
// reset the rows
|
|||
usageDoc.usageQuota.rows = 0 |
|||
|
|||
// sync the apps
|
|||
const apps = await getAllApps(CouchDB, { dev: true }) |
|||
const appCount = apps ? apps.length : 0 |
|||
usageDoc.usageQuota.apps = appCount |
|||
|
|||
await db.put(usageDoc) |
|||
} |
|||
) |
|||
} |
|||
@ -0,0 +1,27 @@ |
|||
const env = require("../../../environment") |
|||
const TestConfig = require("../../../tests/utilities/TestConfiguration") |
|||
|
|||
const syncApps = jest.fn() |
|||
const syncRows = jest.fn() |
|||
|
|||
jest.mock("../../usageQuotas/syncApps", () => ({ run: syncApps }) ) |
|||
jest.mock("../../usageQuotas/syncRows", () => ({ run: syncRows }) ) |
|||
|
|||
const migrations = require("../../usageQuotas") |
|||
|
|||
describe("run", () => { |
|||
let config = new TestConfig(false) |
|||
|
|||
beforeEach(async () => { |
|||
await config.init() |
|||
env._set("USE_QUOTAS", 1) |
|||
}) |
|||
|
|||
afterAll(config.end) |
|||
|
|||
it("runs the required migrations", async () => { |
|||
await migrations.run() |
|||
expect(syncApps).toHaveBeenCalledTimes(1) |
|||
expect(syncRows).toHaveBeenCalledTimes(1) |
|||
}) |
|||
}) |
|||
@ -0,0 +1,43 @@ |
|||
const { getGlobalDB } = require("@budibase/backend-core/tenancy") |
|||
const TestConfig = require("../../../tests/utilities/TestConfiguration") |
|||
const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota") |
|||
const syncRows = require("../../usageQuotas/syncRows") |
|||
const env = require("../../../environment") |
|||
|
|||
describe("syncRows", () => { |
|||
let config = new TestConfig(false) |
|||
|
|||
beforeEach(async () => { |
|||
await config.init() |
|||
env._set("USE_QUOTAS", 1) |
|||
}) |
|||
|
|||
afterAll(config.end) |
|||
|
|||
it("runs successfully", async () => { |
|||
// create the usage quota doc and mock usages
|
|||
const db = getGlobalDB() |
|||
await getUsageQuotaDoc(db) |
|||
await update(Properties.ROW, 300) |
|||
|
|||
let usageDoc = await getUsageQuotaDoc(db) |
|||
expect(usageDoc.usageQuota.rows).toEqual(300) |
|||
|
|||
// app 1
|
|||
await config.createTable() |
|||
await config.createRow() |
|||
// app 2
|
|||
await config.createApp() |
|||
await config.createTable() |
|||
await config.createRow() |
|||
await config.createRow() |
|||
|
|||
// migrate
|
|||
await syncRows.run() |
|||
|
|||
// assert the migration worked
|
|||
usageDoc = await getUsageQuotaDoc(db) |
|||
expect(usageDoc.usageQuota.rows).toEqual(3) |
|||
}) |
|||
}) |
|||
|
|||
@ -0,0 +1,24 @@ |
|||
const { |
|||
MIGRATIONS, |
|||
MIGRATION_DBS, |
|||
migrateIfRequired, |
|||
} = require("@budibase/backend-core/migrations") |
|||
const { useQuotas } = require("../../utilities/usageQuota") |
|||
const syncApps = require("./syncApps") |
|||
const syncRows = require("./syncRows") |
|||
|
|||
exports.run = async () => { |
|||
if (!useQuotas()) { |
|||
return |
|||
} |
|||
|
|||
// Jan 2022
|
|||
await migrateIfRequired( |
|||
MIGRATION_DBS.GLOBAL_DB, |
|||
MIGRATIONS.QUOTAS_1, |
|||
async () => { |
|||
await syncApps.run() |
|||
await syncRows.run() |
|||
} |
|||
) |
|||
} |
|||
@ -0,0 +1,18 @@ |
|||
const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy") |
|||
const { getAllApps } = require("@budibase/backend-core/db") |
|||
const CouchDB = require("../../db") |
|||
const { getUsageQuotaDoc } = require("../../utilities/usageQuota") |
|||
|
|||
exports.run = async () => { |
|||
const db = getGlobalDB() |
|||
// get app count
|
|||
const devApps = await getAllApps(CouchDB, { dev: true }) |
|||
const appCount = devApps ? devApps.length : 0 |
|||
|
|||
// sync app count
|
|||
const tenantId = getTenantId() |
|||
console.log(`[Tenant: ${tenantId}] Syncing app count: ${appCount}`) |
|||
const usageDoc = await getUsageQuotaDoc(db) |
|||
usageDoc.usageQuota.apps = appCount |
|||
await db.put(usageDoc) |
|||
} |
|||
@ -0,0 +1,21 @@ |
|||
const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy") |
|||
const { getAllApps } = require("@budibase/backend-core/db") |
|||
const CouchDB = require("../../db") |
|||
const { getUsageQuotaDoc } = require("../../utilities/usageQuota") |
|||
const { getUniqueRows } = require("../../utilities/usageQuota/rows") |
|||
|
|||
exports.run = async () => { |
|||
const db = getGlobalDB() |
|||
// get all rows in all apps
|
|||
const allApps = await getAllApps(CouchDB, { all: true }) |
|||
const appIds = allApps ? allApps.map(app => app.appId) : [] |
|||
const rows = await getUniqueRows(appIds) |
|||
const rowCount = rows ? rows.length : 0 |
|||
|
|||
// sync row count
|
|||
const tenantId = getTenantId() |
|||
console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`) |
|||
const usageDoc = await getUsageQuotaDoc(db) |
|||
usageDoc.usageQuota.rows = rowCount |
|||
await db.put(usageDoc) |
|||
} |
|||
@ -0,0 +1,72 @@ |
|||
const getTenantId = jest.fn() |
|||
jest.mock("@budibase/backend-core/tenancy", () => ({ |
|||
getTenantId |
|||
})) |
|||
const usageQuota = require("../../usageQuota") |
|||
const env = require("../../../environment") |
|||
|
|||
class TestConfiguration { |
|||
constructor() { |
|||
this.enableQuotas() |
|||
} |
|||
|
|||
enableQuotas = () => { |
|||
env.USE_QUOTAS = 1 |
|||
} |
|||
|
|||
disableQuotas = () => { |
|||
env.USE_QUOTAS = null |
|||
} |
|||
|
|||
setTenantId = (tenantId) => { |
|||
getTenantId.mockReturnValue(tenantId) |
|||
} |
|||
|
|||
setExcludedTenants = (tenants) => { |
|||
env.EXCLUDE_QUOTAS_TENANTS = tenants |
|||
} |
|||
|
|||
reset = () => { |
|||
this.disableQuotas() |
|||
this.setExcludedTenants(null) |
|||
} |
|||
} |
|||
|
|||
describe("usageQuota", () => { |
|||
let config |
|||
|
|||
beforeEach(() => { |
|||
config = new TestConfiguration() |
|||
}) |
|||
|
|||
afterEach(() => { |
|||
config.reset() |
|||
}) |
|||
|
|||
describe("useQuotas", () => { |
|||
it("works when no settings have been provided", () => { |
|||
config.reset() |
|||
expect(usageQuota.useQuotas()).toBe(false) |
|||
}) |
|||
it("honours USE_QUOTAS setting", () => { |
|||
config.disableQuotas() |
|||
expect(usageQuota.useQuotas()).toBe(false) |
|||
|
|||
config.enableQuotas() |
|||
expect(usageQuota.useQuotas()).toBe(true) |
|||
}) |
|||
it("honours EXCLUDE_QUOTAS_TENANTS setting", () => { |
|||
config.setTenantId("test") |
|||
|
|||
// tenantId is in the list
|
|||
config.setExcludedTenants("test, test2, test2") |
|||
expect(usageQuota.useQuotas()).toBe(false) |
|||
config.setExcludedTenants("test,test2,test2") |
|||
expect(usageQuota.useQuotas()).toBe(false) |
|||
|
|||
// tenantId is not in the list
|
|||
config.setTenantId("other") |
|||
expect(usageQuota.useQuotas()).toBe(true) |
|||
}) |
|||
}) |
|||
}) |
|||
@ -0,0 +1,74 @@ |
|||
const { getRowParams, USER_METDATA_PREFIX } = require("../../db/utils") |
|||
const CouchDB = require("../../db") |
|||
const { isDevAppID, getDevelopmentAppID } = require("@budibase/backend-core/db") |
|||
|
|||
const ROW_EXCLUSIONS = [USER_METDATA_PREFIX] |
|||
|
|||
const getAppPairs = appIds => { |
|||
// collect the app ids into dev / prod pairs
|
|||
// keyed by the dev app id
|
|||
const pairs = {} |
|||
for (let appId of appIds) { |
|||
const devId = getDevelopmentAppID(appId) |
|||
if (!pairs[devId]) { |
|||
pairs[devId] = {} |
|||
} |
|||
if (isDevAppID(appId)) { |
|||
pairs[devId].devId = appId |
|||
} else { |
|||
pairs[devId].prodId = appId |
|||
} |
|||
} |
|||
return pairs |
|||
} |
|||
|
|||
const getAppRows = async appId => { |
|||
const appDb = new CouchDB(appId) |
|||
const response = await appDb.allDocs( |
|||
getRowParams(null, null, { |
|||
include_docs: false, |
|||
}) |
|||
) |
|||
return response.rows |
|||
.map(r => r.id) |
|||
.filter(id => { |
|||
for (let exclusion of ROW_EXCLUSIONS) { |
|||
if (id.startsWith(exclusion)) { |
|||
return false |
|||
} |
|||
} |
|||
return true |
|||
}) |
|||
} |
|||
|
|||
/** |
|||
* Return a set of all rows in the given app ids. |
|||
* The returned rows will be unique on a per dev/prod app basis. |
|||
* Rows duplicates may exist across apps due to data import so they are not filtered out. |
|||
*/ |
|||
exports.getUniqueRows = async appIds => { |
|||
let uniqueRows = [] |
|||
const pairs = getAppPairs(appIds) |
|||
|
|||
for (let pair of Object.values(pairs)) { |
|||
let appRows = [] |
|||
for (let appId of [pair.devId, pair.prodId]) { |
|||
if (!appId) { |
|||
continue |
|||
} |
|||
try { |
|||
appRows.push(await getAppRows(appId)) |
|||
} catch (e) { |
|||
console.error(e) |
|||
// don't error out if we can't count the app rows, just continue
|
|||
} |
|||
} |
|||
|
|||
// ensure uniqueness on a per app pair basis
|
|||
// this can't be done on all rows because app import results in
|
|||
// duplicate row ids across apps
|
|||
uniqueRows = uniqueRows.concat(...new Set(appRows)) |
|||
} |
|||
|
|||
return uniqueRows |
|||
} |
|||
Loading…
Reference in new issue