forked from tsai/budibase
Rory Powell
5 years ago
committed by
GitHub
GitHub
23 changed files with 462 additions and 103 deletions
@ -1,30 +0,0 @@ |
|||||
const { |
|
||||
MIGRATIONS, |
|
||||
MIGRATION_DBS, |
|
||||
migrateIfRequired, |
|
||||
} = require("@budibase/backend-core/migrations") |
|
||||
const { getGlobalDB } = require("@budibase/backend-core/tenancy") |
|
||||
const { getAllApps } = require("@budibase/backend-core/db") |
|
||||
const CouchDB = require("../db") |
|
||||
const { getUsageQuotaDoc } = require("../utilities/usageQuota") |
|
||||
|
|
||||
exports.runIfRequired = async () => { |
|
||||
await migrateIfRequired( |
|
||||
MIGRATION_DBS.GLOBAL_DB, |
|
||||
MIGRATIONS.SYNC_APP_AND_RESET_ROWS_QUOTAS, |
|
||||
async () => { |
|
||||
const db = getGlobalDB() |
|
||||
const usageDoc = await getUsageQuotaDoc(db) |
|
||||
|
|
||||
// reset the rows
|
|
||||
usageDoc.usageQuota.rows = 0 |
|
||||
|
|
||||
// sync the apps
|
|
||||
const apps = await getAllApps(CouchDB, { dev: true }) |
|
||||
const appCount = apps ? apps.length : 0 |
|
||||
usageDoc.usageQuota.apps = appCount |
|
||||
|
|
||||
await db.put(usageDoc) |
|
||||
} |
|
||||
) |
|
||||
} |
|
||||
@ -0,0 +1,27 @@ |
|||||
|
const env = require("../../../environment") |
||||
|
const TestConfig = require("../../../tests/utilities/TestConfiguration") |
||||
|
|
||||
|
const syncApps = jest.fn() |
||||
|
const syncRows = jest.fn() |
||||
|
|
||||
|
jest.mock("../../usageQuotas/syncApps", () => ({ run: syncApps }) ) |
||||
|
jest.mock("../../usageQuotas/syncRows", () => ({ run: syncRows }) ) |
||||
|
|
||||
|
const migrations = require("../../usageQuotas") |
||||
|
|
||||
|
describe("run", () => { |
||||
|
let config = new TestConfig(false) |
||||
|
|
||||
|
beforeEach(async () => { |
||||
|
await config.init() |
||||
|
env._set("USE_QUOTAS", 1) |
||||
|
}) |
||||
|
|
||||
|
afterAll(config.end) |
||||
|
|
||||
|
it("runs the required migrations", async () => { |
||||
|
await migrations.run() |
||||
|
expect(syncApps).toHaveBeenCalledTimes(1) |
||||
|
expect(syncRows).toHaveBeenCalledTimes(1) |
||||
|
}) |
||||
|
}) |
||||
@ -0,0 +1,43 @@ |
|||||
|
const { getGlobalDB } = require("@budibase/backend-core/tenancy") |
||||
|
const TestConfig = require("../../../tests/utilities/TestConfiguration") |
||||
|
const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota") |
||||
|
const syncRows = require("../../usageQuotas/syncRows") |
||||
|
const env = require("../../../environment") |
||||
|
|
||||
|
describe("syncRows", () => { |
||||
|
let config = new TestConfig(false) |
||||
|
|
||||
|
beforeEach(async () => { |
||||
|
await config.init() |
||||
|
env._set("USE_QUOTAS", 1) |
||||
|
}) |
||||
|
|
||||
|
afterAll(config.end) |
||||
|
|
||||
|
it("runs successfully", async () => { |
||||
|
// create the usage quota doc and mock usages
|
||||
|
const db = getGlobalDB() |
||||
|
await getUsageQuotaDoc(db) |
||||
|
await update(Properties.ROW, 300) |
||||
|
|
||||
|
let usageDoc = await getUsageQuotaDoc(db) |
||||
|
expect(usageDoc.usageQuota.rows).toEqual(300) |
||||
|
|
||||
|
// app 1
|
||||
|
await config.createTable() |
||||
|
await config.createRow() |
||||
|
// app 2
|
||||
|
await config.createApp() |
||||
|
await config.createTable() |
||||
|
await config.createRow() |
||||
|
await config.createRow() |
||||
|
|
||||
|
// migrate
|
||||
|
await syncRows.run() |
||||
|
|
||||
|
// assert the migration worked
|
||||
|
usageDoc = await getUsageQuotaDoc(db) |
||||
|
expect(usageDoc.usageQuota.rows).toEqual(3) |
||||
|
}) |
||||
|
}) |
||||
|
|
||||
@ -0,0 +1,24 @@ |
|||||
|
const { |
||||
|
MIGRATIONS, |
||||
|
MIGRATION_DBS, |
||||
|
migrateIfRequired, |
||||
|
} = require("@budibase/backend-core/migrations") |
||||
|
const { useQuotas } = require("../../utilities/usageQuota") |
||||
|
const syncApps = require("./syncApps") |
||||
|
const syncRows = require("./syncRows") |
||||
|
|
||||
|
exports.run = async () => { |
||||
|
if (!useQuotas()) { |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
// Jan 2022
|
||||
|
await migrateIfRequired( |
||||
|
MIGRATION_DBS.GLOBAL_DB, |
||||
|
MIGRATIONS.QUOTAS_1, |
||||
|
async () => { |
||||
|
await syncApps.run() |
||||
|
await syncRows.run() |
||||
|
} |
||||
|
) |
||||
|
} |
||||
@ -0,0 +1,18 @@ |
|||||
|
const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy") |
||||
|
const { getAllApps } = require("@budibase/backend-core/db") |
||||
|
const CouchDB = require("../../db") |
||||
|
const { getUsageQuotaDoc } = require("../../utilities/usageQuota") |
||||
|
|
||||
|
exports.run = async () => { |
||||
|
const db = getGlobalDB() |
||||
|
// get app count
|
||||
|
const devApps = await getAllApps(CouchDB, { dev: true }) |
||||
|
const appCount = devApps ? devApps.length : 0 |
||||
|
|
||||
|
// sync app count
|
||||
|
const tenantId = getTenantId() |
||||
|
console.log(`[Tenant: ${tenantId}] Syncing app count: ${appCount}`) |
||||
|
const usageDoc = await getUsageQuotaDoc(db) |
||||
|
usageDoc.usageQuota.apps = appCount |
||||
|
await db.put(usageDoc) |
||||
|
} |
||||
@ -0,0 +1,21 @@ |
|||||
|
const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy") |
||||
|
const { getAllApps } = require("@budibase/backend-core/db") |
||||
|
const CouchDB = require("../../db") |
||||
|
const { getUsageQuotaDoc } = require("../../utilities/usageQuota") |
||||
|
const { getUniqueRows } = require("../../utilities/usageQuota/rows") |
||||
|
|
||||
|
exports.run = async () => { |
||||
|
const db = getGlobalDB() |
||||
|
// get all rows in all apps
|
||||
|
const allApps = await getAllApps(CouchDB, { all: true }) |
||||
|
const appIds = allApps ? allApps.map(app => app.appId) : [] |
||||
|
const rows = await getUniqueRows(appIds) |
||||
|
const rowCount = rows ? rows.length : 0 |
||||
|
|
||||
|
// sync row count
|
||||
|
const tenantId = getTenantId() |
||||
|
console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`) |
||||
|
const usageDoc = await getUsageQuotaDoc(db) |
||||
|
usageDoc.usageQuota.rows = rowCount |
||||
|
await db.put(usageDoc) |
||||
|
} |
||||
@ -0,0 +1,72 @@ |
|||||
|
const getTenantId = jest.fn() |
||||
|
jest.mock("@budibase/backend-core/tenancy", () => ({ |
||||
|
getTenantId |
||||
|
})) |
||||
|
const usageQuota = require("../../usageQuota") |
||||
|
const env = require("../../../environment") |
||||
|
|
||||
|
class TestConfiguration { |
||||
|
constructor() { |
||||
|
this.enableQuotas() |
||||
|
} |
||||
|
|
||||
|
enableQuotas = () => { |
||||
|
env.USE_QUOTAS = 1 |
||||
|
} |
||||
|
|
||||
|
disableQuotas = () => { |
||||
|
env.USE_QUOTAS = null |
||||
|
} |
||||
|
|
||||
|
setTenantId = (tenantId) => { |
||||
|
getTenantId.mockReturnValue(tenantId) |
||||
|
} |
||||
|
|
||||
|
setExcludedTenants = (tenants) => { |
||||
|
env.EXCLUDE_QUOTAS_TENANTS = tenants |
||||
|
} |
||||
|
|
||||
|
reset = () => { |
||||
|
this.disableQuotas() |
||||
|
this.setExcludedTenants(null) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
describe("usageQuota", () => { |
||||
|
let config |
||||
|
|
||||
|
beforeEach(() => { |
||||
|
config = new TestConfiguration() |
||||
|
}) |
||||
|
|
||||
|
afterEach(() => { |
||||
|
config.reset() |
||||
|
}) |
||||
|
|
||||
|
describe("useQuotas", () => { |
||||
|
it("works when no settings have been provided", () => { |
||||
|
config.reset() |
||||
|
expect(usageQuota.useQuotas()).toBe(false) |
||||
|
}) |
||||
|
it("honours USE_QUOTAS setting", () => { |
||||
|
config.disableQuotas() |
||||
|
expect(usageQuota.useQuotas()).toBe(false) |
||||
|
|
||||
|
config.enableQuotas() |
||||
|
expect(usageQuota.useQuotas()).toBe(true) |
||||
|
}) |
||||
|
it("honours EXCLUDE_QUOTAS_TENANTS setting", () => { |
||||
|
config.setTenantId("test") |
||||
|
|
||||
|
// tenantId is in the list
|
||||
|
config.setExcludedTenants("test, test2, test2") |
||||
|
expect(usageQuota.useQuotas()).toBe(false) |
||||
|
config.setExcludedTenants("test,test2,test2") |
||||
|
expect(usageQuota.useQuotas()).toBe(false) |
||||
|
|
||||
|
// tenantId is not in the list
|
||||
|
config.setTenantId("other") |
||||
|
expect(usageQuota.useQuotas()).toBe(true) |
||||
|
}) |
||||
|
}) |
||||
|
}) |
||||
@ -0,0 +1,74 @@ |
|||||
|
const { getRowParams, USER_METDATA_PREFIX } = require("../../db/utils") |
||||
|
const CouchDB = require("../../db") |
||||
|
const { isDevAppID, getDevelopmentAppID } = require("@budibase/backend-core/db") |
||||
|
|
||||
|
const ROW_EXCLUSIONS = [USER_METDATA_PREFIX] |
||||
|
|
||||
|
const getAppPairs = appIds => { |
||||
|
// collect the app ids into dev / prod pairs
|
||||
|
// keyed by the dev app id
|
||||
|
const pairs = {} |
||||
|
for (let appId of appIds) { |
||||
|
const devId = getDevelopmentAppID(appId) |
||||
|
if (!pairs[devId]) { |
||||
|
pairs[devId] = {} |
||||
|
} |
||||
|
if (isDevAppID(appId)) { |
||||
|
pairs[devId].devId = appId |
||||
|
} else { |
||||
|
pairs[devId].prodId = appId |
||||
|
} |
||||
|
} |
||||
|
return pairs |
||||
|
} |
||||
|
|
||||
|
const getAppRows = async appId => { |
||||
|
const appDb = new CouchDB(appId) |
||||
|
const response = await appDb.allDocs( |
||||
|
getRowParams(null, null, { |
||||
|
include_docs: false, |
||||
|
}) |
||||
|
) |
||||
|
return response.rows |
||||
|
.map(r => r.id) |
||||
|
.filter(id => { |
||||
|
for (let exclusion of ROW_EXCLUSIONS) { |
||||
|
if (id.startsWith(exclusion)) { |
||||
|
return false |
||||
|
} |
||||
|
} |
||||
|
return true |
||||
|
}) |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* Return a set of all rows in the given app ids. |
||||
|
* The returned rows will be unique on a per dev/prod app basis. |
||||
|
* Rows duplicates may exist across apps due to data import so they are not filtered out. |
||||
|
*/ |
||||
|
exports.getUniqueRows = async appIds => { |
||||
|
let uniqueRows = [] |
||||
|
const pairs = getAppPairs(appIds) |
||||
|
|
||||
|
for (let pair of Object.values(pairs)) { |
||||
|
let appRows = [] |
||||
|
for (let appId of [pair.devId, pair.prodId]) { |
||||
|
if (!appId) { |
||||
|
continue |
||||
|
} |
||||
|
try { |
||||
|
appRows.push(await getAppRows(appId)) |
||||
|
} catch (e) { |
||||
|
console.error(e) |
||||
|
// don't error out if we can't count the app rows, just continue
|
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// ensure uniqueness on a per app pair basis
|
||||
|
// this can't be done on all rows because app import results in
|
||||
|
// duplicate row ids across apps
|
||||
|
uniqueRows = uniqueRows.concat(...new Set(appRows)) |
||||
|
} |
||||
|
|
||||
|
return uniqueRows |
||||
|
} |
||||
Loading…
Reference in new issue