forked from tsai/budibase
Rory Powell
5 years ago
45 changed files with 678 additions and 185 deletions
@ -0,0 +1 @@ |
|||
module.exports = require("./src/migrations") |
|||
@ -0,0 +1,27 @@ |
|||
const env = require("../../../environment") |
|||
const TestConfig = require("../../../tests/utilities/TestConfiguration") |
|||
|
|||
const syncApps = jest.fn() |
|||
const syncRows = jest.fn() |
|||
|
|||
jest.mock("../../usageQuotas/syncApps", () => ({ run: syncApps }) ) |
|||
jest.mock("../../usageQuotas/syncRows", () => ({ run: syncRows }) ) |
|||
|
|||
const migrations = require("../../usageQuotas") |
|||
|
|||
describe("run", () => { |
|||
let config = new TestConfig(false) |
|||
|
|||
beforeEach(async () => { |
|||
await config.init() |
|||
env._set("USE_QUOTAS", 1) |
|||
}) |
|||
|
|||
afterAll(config.end) |
|||
|
|||
it("runs the required migrations", async () => { |
|||
await migrations.run() |
|||
expect(syncApps).toHaveBeenCalledTimes(1) |
|||
expect(syncRows).toHaveBeenCalledTimes(1) |
|||
}) |
|||
}) |
|||
@ -0,0 +1,37 @@ |
|||
const { getGlobalDB } = require("@budibase/backend-core/tenancy") |
|||
const TestConfig = require("../../../tests/utilities/TestConfiguration") |
|||
const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota") |
|||
const syncApps = require("../../usageQuotas/syncApps") |
|||
const env = require("../../../environment") |
|||
|
|||
describe("syncApps", () => { |
|||
let config = new TestConfig(false) |
|||
|
|||
beforeEach(async () => { |
|||
await config.init() |
|||
env._set("USE_QUOTAS", 1) |
|||
}) |
|||
|
|||
afterAll(config.end) |
|||
|
|||
it("runs successfully", async () => { |
|||
// create the usage quota doc and mock usages
|
|||
const db = getGlobalDB() |
|||
await getUsageQuotaDoc(db) |
|||
await update(Properties.APPS, 3) |
|||
|
|||
let usageDoc = await getUsageQuotaDoc(db) |
|||
expect(usageDoc.usageQuota.apps).toEqual(3) |
|||
|
|||
// create an extra app to test the migration
|
|||
await config.createApp("quota-test") |
|||
|
|||
// migrate
|
|||
await syncApps.run() |
|||
|
|||
// assert the migration worked
|
|||
usageDoc = await getUsageQuotaDoc(db) |
|||
expect(usageDoc.usageQuota.apps).toEqual(2) |
|||
}) |
|||
}) |
|||
|
|||
@ -0,0 +1,43 @@ |
|||
const { getGlobalDB } = require("@budibase/backend-core/tenancy") |
|||
const TestConfig = require("../../../tests/utilities/TestConfiguration") |
|||
const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota") |
|||
const syncRows = require("../../usageQuotas/syncRows") |
|||
const env = require("../../../environment") |
|||
|
|||
describe("syncRows", () => { |
|||
let config = new TestConfig(false) |
|||
|
|||
beforeEach(async () => { |
|||
await config.init() |
|||
env._set("USE_QUOTAS", 1) |
|||
}) |
|||
|
|||
afterAll(config.end) |
|||
|
|||
it("runs successfully", async () => { |
|||
// create the usage quota doc and mock usages
|
|||
const db = getGlobalDB() |
|||
await getUsageQuotaDoc(db) |
|||
await update(Properties.ROW, 300) |
|||
|
|||
let usageDoc = await getUsageQuotaDoc(db) |
|||
expect(usageDoc.usageQuota.rows).toEqual(300) |
|||
|
|||
// app 1
|
|||
await config.createTable() |
|||
await config.createRow() |
|||
// app 2
|
|||
await config.createApp() |
|||
await config.createTable() |
|||
await config.createRow() |
|||
await config.createRow() |
|||
|
|||
// migrate
|
|||
await syncRows.run() |
|||
|
|||
// assert the migration worked
|
|||
usageDoc = await getUsageQuotaDoc(db) |
|||
expect(usageDoc.usageQuota.rows).toEqual(3) |
|||
}) |
|||
}) |
|||
|
|||
@ -0,0 +1,24 @@ |
|||
const { |
|||
MIGRATIONS, |
|||
MIGRATION_DBS, |
|||
migrateIfRequired, |
|||
} = require("@budibase/backend-core/migrations") |
|||
const { useQuotas } = require("../../utilities/usageQuota") |
|||
const syncApps = require("./syncApps") |
|||
const syncRows = require("./syncRows") |
|||
|
|||
exports.run = async () => { |
|||
if (!useQuotas()) { |
|||
return |
|||
} |
|||
|
|||
// Jan 2022
|
|||
await migrateIfRequired( |
|||
MIGRATION_DBS.GLOBAL_DB, |
|||
MIGRATIONS.QUOTAS_1, |
|||
async () => { |
|||
await syncApps.run() |
|||
await syncRows.run() |
|||
} |
|||
) |
|||
} |
|||
@ -0,0 +1,18 @@ |
|||
const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy") |
|||
const { getAllApps } = require("@budibase/backend-core/db") |
|||
const CouchDB = require("../../db") |
|||
const { getUsageQuotaDoc } = require("../../utilities/usageQuota") |
|||
|
|||
exports.run = async () => { |
|||
const db = getGlobalDB() |
|||
// get app count
|
|||
const devApps = await getAllApps(CouchDB, { dev: true }) |
|||
const appCount = devApps ? devApps.length : 0 |
|||
|
|||
// sync app count
|
|||
const tenantId = getTenantId() |
|||
console.log(`[Tenant: ${tenantId}] Syncing app count: ${appCount}`) |
|||
const usageDoc = await getUsageQuotaDoc(db) |
|||
usageDoc.usageQuota.apps = appCount |
|||
await db.put(usageDoc) |
|||
} |
|||
@ -0,0 +1,21 @@ |
|||
const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy") |
|||
const { getAllApps } = require("@budibase/backend-core/db") |
|||
const CouchDB = require("../../db") |
|||
const { getUsageQuotaDoc } = require("../../utilities/usageQuota") |
|||
const { getUniqueRows } = require("../../utilities/usageQuota/rows") |
|||
|
|||
exports.run = async () => { |
|||
const db = getGlobalDB() |
|||
// get all rows in all apps
|
|||
const allApps = await getAllApps(CouchDB, { all: true }) |
|||
const appIds = allApps ? allApps.map(app => app.appId) : [] |
|||
const rows = await getUniqueRows(appIds) |
|||
const rowCount = rows ? rows.length : 0 |
|||
|
|||
// sync row count
|
|||
const tenantId = getTenantId() |
|||
console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`) |
|||
const usageDoc = await getUsageQuotaDoc(db) |
|||
usageDoc.usageQuota.rows = rowCount |
|||
await db.put(usageDoc) |
|||
} |
|||
@ -0,0 +1,72 @@ |
|||
const getTenantId = jest.fn() |
|||
jest.mock("@budibase/backend-core/tenancy", () => ({ |
|||
getTenantId |
|||
})) |
|||
const usageQuota = require("../../usageQuota") |
|||
const env = require("../../../environment") |
|||
|
|||
class TestConfiguration { |
|||
constructor() { |
|||
this.enableQuotas() |
|||
} |
|||
|
|||
enableQuotas = () => { |
|||
env.USE_QUOTAS = 1 |
|||
} |
|||
|
|||
disableQuotas = () => { |
|||
env.USE_QUOTAS = null |
|||
} |
|||
|
|||
setTenantId = (tenantId) => { |
|||
getTenantId.mockReturnValue(tenantId) |
|||
} |
|||
|
|||
setExcludedTenants = (tenants) => { |
|||
env.EXCLUDE_QUOTAS_TENANTS = tenants |
|||
} |
|||
|
|||
reset = () => { |
|||
this.disableQuotas() |
|||
this.setExcludedTenants(null) |
|||
} |
|||
} |
|||
|
|||
describe("usageQuota", () => { |
|||
let config |
|||
|
|||
beforeEach(() => { |
|||
config = new TestConfiguration() |
|||
}) |
|||
|
|||
afterEach(() => { |
|||
config.reset() |
|||
}) |
|||
|
|||
describe("useQuotas", () => { |
|||
it("works when no settings have been provided", () => { |
|||
config.reset() |
|||
expect(usageQuota.useQuotas()).toBe(false) |
|||
}) |
|||
it("honours USE_QUOTAS setting", () => { |
|||
config.disableQuotas() |
|||
expect(usageQuota.useQuotas()).toBe(false) |
|||
|
|||
config.enableQuotas() |
|||
expect(usageQuota.useQuotas()).toBe(true) |
|||
}) |
|||
it("honours EXCLUDE_QUOTAS_TENANTS setting", () => { |
|||
config.setTenantId("test") |
|||
|
|||
// tenantId is in the list
|
|||
config.setExcludedTenants("test, test2, test2") |
|||
expect(usageQuota.useQuotas()).toBe(false) |
|||
config.setExcludedTenants("test,test2,test2") |
|||
expect(usageQuota.useQuotas()).toBe(false) |
|||
|
|||
// tenantId is not in the list
|
|||
config.setTenantId("other") |
|||
expect(usageQuota.useQuotas()).toBe(true) |
|||
}) |
|||
}) |
|||
}) |
|||
@ -1,73 +0,0 @@ |
|||
const env = require("../environment") |
|||
const { getGlobalDB } = require("@budibase/backend-core/tenancy") |
|||
const { |
|||
StaticDatabases, |
|||
generateNewUsageQuotaDoc, |
|||
} = require("@budibase/backend-core/db") |
|||
|
|||
function getNewQuotaReset() { |
|||
return Date.now() + 2592000000 |
|||
} |
|||
|
|||
exports.Properties = { |
|||
ROW: "rows", |
|||
UPLOAD: "storage", |
|||
VIEW: "views", |
|||
USER: "users", |
|||
AUTOMATION: "automationRuns", |
|||
APPS: "apps", |
|||
EMAILS: "emails", |
|||
} |
|||
|
|||
async function getUsageQuotaDoc(db) { |
|||
let quota |
|||
try { |
|||
quota = await db.get(StaticDatabases.PLATFORM_INFO.docs.usageQuota) |
|||
} catch (err) { |
|||
// doc doesn't exist. Create it
|
|||
quota = await db.post(generateNewUsageQuotaDoc()) |
|||
} |
|||
|
|||
return quota |
|||
} |
|||
|
|||
/** |
|||
* Given a specified tenantId this will add to the usage object for the specified property. |
|||
* @param {string} property The property which is to be added to (within the nested usageQuota object). |
|||
* @param {number} usage The amount (this can be negative) to adjust the number by. |
|||
* @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have |
|||
* also been reset after this call. |
|||
*/ |
|||
exports.update = async (property, usage) => { |
|||
if (!env.USE_QUOTAS) { |
|||
return |
|||
} |
|||
|
|||
try { |
|||
const db = getGlobalDB() |
|||
const quota = await getUsageQuotaDoc(db) |
|||
|
|||
// Check if the quota needs reset
|
|||
if (Date.now() >= quota.quotaReset) { |
|||
quota.quotaReset = getNewQuotaReset() |
|||
for (let prop of Object.keys(quota.usageQuota)) { |
|||
quota.usageQuota[prop] = 0 |
|||
} |
|||
} |
|||
|
|||
// increment the quota
|
|||
quota.usageQuota[property] += usage |
|||
|
|||
if (quota.usageQuota[property] > quota.usageLimits[property]) { |
|||
throw new Error( |
|||
`You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.` |
|||
) |
|||
} |
|||
|
|||
// update the usage quotas
|
|||
await db.put(quota) |
|||
} catch (err) { |
|||
console.error(`Error updating usage quotas for ${property}`, err) |
|||
throw err |
|||
} |
|||
} |
|||
@ -0,0 +1,92 @@ |
|||
const env = require("../../environment") |
|||
const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy") |
|||
const { |
|||
StaticDatabases, |
|||
generateNewUsageQuotaDoc, |
|||
} = require("@budibase/backend-core/db") |
|||
|
|||
exports.useQuotas = () => { |
|||
// check if quotas are enabled
|
|||
if (env.USE_QUOTAS) { |
|||
// check if there are any tenants without limits
|
|||
if (env.EXCLUDE_QUOTAS_TENANTS) { |
|||
const excludedTenants = env.EXCLUDE_QUOTAS_TENANTS.replace( |
|||
/\s/g, |
|||
"" |
|||
).split(",") |
|||
const tenantId = getTenantId() |
|||
if (excludedTenants.includes(tenantId)) { |
|||
return false |
|||
} |
|||
} |
|||
return true |
|||
} |
|||
return false |
|||
} |
|||
|
|||
exports.Properties = { |
|||
ROW: "rows", |
|||
UPLOAD: "storage", // doesn't work yet
|
|||
VIEW: "views", // doesn't work yet
|
|||
USER: "users", // doesn't work yet
|
|||
AUTOMATION: "automationRuns", // doesn't work yet
|
|||
APPS: "apps", |
|||
EMAILS: "emails", // doesn't work yet
|
|||
} |
|||
|
|||
exports.getUsageQuotaDoc = async db => { |
|||
let quota |
|||
try { |
|||
quota = await db.get(StaticDatabases.GLOBAL.docs.usageQuota) |
|||
} catch (err) { |
|||
// doc doesn't exist. Create it
|
|||
quota = generateNewUsageQuotaDoc() |
|||
const response = await db.put(quota) |
|||
quota._rev = response.rev |
|||
} |
|||
|
|||
return quota |
|||
} |
|||
|
|||
/** |
|||
* Given a specified tenantId this will add to the usage object for the specified property. |
|||
* @param {string} property The property which is to be added to (within the nested usageQuota object). |
|||
* @param {number} usage The amount (this can be negative) to adjust the number by. |
|||
* @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have |
|||
* also been reset after this call. |
|||
*/ |
|||
exports.update = async (property, usage, opts = { dryRun: false }) => { |
|||
if (!exports.useQuotas()) { |
|||
return |
|||
} |
|||
|
|||
try { |
|||
const db = getGlobalDB() |
|||
const quota = await exports.getUsageQuotaDoc(db) |
|||
|
|||
// increment the quota
|
|||
quota.usageQuota[property] += usage |
|||
|
|||
if ( |
|||
quota.usageQuota[property] > quota.usageLimits[property] && |
|||
usage > 0 // allow for decrementing usage when the quota is already exceeded
|
|||
) { |
|||
throw new Error( |
|||
`You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.` |
|||
) |
|||
} |
|||
|
|||
if (quota.usageQuota[property] < 0) { |
|||
// never go negative if the quota has previously been exceeded
|
|||
quota.usageQuota[property] = 0 |
|||
} |
|||
|
|||
// update the usage quotas
|
|||
if (!opts.dryRun) { |
|||
await db.put(quota) |
|||
} |
|||
} catch (err) { |
|||
console.error(`Error updating usage quotas for ${property}`, err) |
|||
throw err |
|||
} |
|||
} |
|||
@ -0,0 +1,74 @@ |
|||
const { getRowParams, USER_METDATA_PREFIX } = require("../../db/utils") |
|||
const CouchDB = require("../../db") |
|||
const { isDevAppID, getDevelopmentAppID } = require("@budibase/backend-core/db") |
|||
|
|||
const ROW_EXCLUSIONS = [USER_METDATA_PREFIX] |
|||
|
|||
const getAppPairs = appIds => { |
|||
// collect the app ids into dev / prod pairs
|
|||
// keyed by the dev app id
|
|||
const pairs = {} |
|||
for (let appId of appIds) { |
|||
const devId = getDevelopmentAppID(appId) |
|||
if (!pairs[devId]) { |
|||
pairs[devId] = {} |
|||
} |
|||
if (isDevAppID(appId)) { |
|||
pairs[devId].devId = appId |
|||
} else { |
|||
pairs[devId].prodId = appId |
|||
} |
|||
} |
|||
return pairs |
|||
} |
|||
|
|||
const getAppRows = async appId => { |
|||
const appDb = new CouchDB(appId) |
|||
const response = await appDb.allDocs( |
|||
getRowParams(null, null, { |
|||
include_docs: false, |
|||
}) |
|||
) |
|||
return response.rows |
|||
.map(r => r.id) |
|||
.filter(id => { |
|||
for (let exclusion of ROW_EXCLUSIONS) { |
|||
if (id.startsWith(exclusion)) { |
|||
return false |
|||
} |
|||
} |
|||
return true |
|||
}) |
|||
} |
|||
|
|||
/** |
|||
* Return a set of all rows in the given app ids. |
|||
* The returned rows will be unique on a per dev/prod app basis. |
|||
* Rows duplicates may exist across apps due to data import so they are not filtered out. |
|||
*/ |
|||
exports.getUniqueRows = async appIds => { |
|||
let uniqueRows = [] |
|||
const pairs = getAppPairs(appIds) |
|||
|
|||
for (let pair of Object.values(pairs)) { |
|||
let appRows = [] |
|||
for (let appId of [pair.devId, pair.prodId]) { |
|||
if (!appId) { |
|||
continue |
|||
} |
|||
try { |
|||
appRows.push(await getAppRows(appId)) |
|||
} catch (e) { |
|||
console.error(e) |
|||
// don't error out if we can't count the app rows, just continue
|
|||
} |
|||
} |
|||
|
|||
// ensure uniqueness on a per app pair basis
|
|||
// this can't be done on all rows because app import results in
|
|||
// duplicate row ids across apps
|
|||
uniqueRows = uniqueRows.concat(...new Set(appRows)) |
|||
} |
|||
|
|||
return uniqueRows |
|||
} |
|||
@ -0,0 +1,18 @@ |
|||
// UNUSED CODE
|
|||
// Preserved for future use
|
|||
|
|||
/* eslint-disable no-unused-vars */ |
|||
|
|||
function getNewQuotaReset() { |
|||
return Date.now() + 2592000000 |
|||
} |
|||
|
|||
function resetQuotasIfRequired(quota) { |
|||
// Check if the quota needs reset
|
|||
if (Date.now() >= quota.quotaReset) { |
|||
quota.quotaReset = getNewQuotaReset() |
|||
for (let prop of Object.keys(quota.usageQuota)) { |
|||
quota.usageQuota[prop] = 0 |
|||
} |
|||
} |
|||
} |
|||
Loading…
Reference in new issue