forked from tsai/budibase
mike12345567
5 years ago
50 changed files with 793 additions and 317 deletions
@ -0,0 +1 @@ |
|||||
|
module.exports = require("./src/migrations") |
||||
@ -1,69 +0,0 @@ |
|||||
<script> |
|
||||
import { Heading, Layout, Icon } from "@budibase/bbui" |
|
||||
|
|
||||
export let onSelect |
|
||||
</script> |
|
||||
|
|
||||
<Layout gap="XS" noPadding> |
|
||||
<div class="template start-from-scratch" on:click={() => onSelect(null)}> |
|
||||
<div |
|
||||
class="background-icon" |
|
||||
style={`background: rgb(50, 50, 50); color: white;`} |
|
||||
> |
|
||||
<Icon name="Add" /> |
|
||||
</div> |
|
||||
<Heading size="XS">Start from scratch</Heading> |
|
||||
<p class="detail">BLANK</p> |
|
||||
</div> |
|
||||
<div |
|
||||
class="template import" |
|
||||
on:click={() => onSelect(null, { useImport: true })} |
|
||||
> |
|
||||
<div |
|
||||
class="background-icon" |
|
||||
style={`background: rgb(50, 50, 50); color: white;`} |
|
||||
> |
|
||||
<Icon name="Add" /> |
|
||||
</div> |
|
||||
<Heading size="XS">Import an app</Heading> |
|
||||
<p class="detail">BLANK</p> |
|
||||
</div> |
|
||||
</Layout> |
|
||||
|
|
||||
<style> |
|
||||
.background-icon { |
|
||||
padding: 10px; |
|
||||
border-radius: 4px; |
|
||||
display: flex; |
|
||||
align-items: center; |
|
||||
justify-content: center; |
|
||||
width: 18px; |
|
||||
color: white; |
|
||||
} |
|
||||
|
|
||||
.template { |
|
||||
min-height: 60px; |
|
||||
display: grid; |
|
||||
grid-gap: var(--layout-s); |
|
||||
grid-template-columns: auto 1fr auto; |
|
||||
border: 1px solid #494949; |
|
||||
align-items: center; |
|
||||
cursor: pointer; |
|
||||
border-radius: 4px; |
|
||||
background: var(--background-alt); |
|
||||
padding: 8px 16px; |
|
||||
} |
|
||||
|
|
||||
.detail { |
|
||||
text-align: right; |
|
||||
} |
|
||||
|
|
||||
.start-from-scratch { |
|
||||
background: var(--spectrum-global-color-gray-50); |
|
||||
margin-top: 20px; |
|
||||
} |
|
||||
|
|
||||
.import { |
|
||||
background: var(--spectrum-global-color-gray-50); |
|
||||
} |
|
||||
</style> |
|
||||
@ -0,0 +1,27 @@ |
|||||
|
const env = require("../../../environment") |
||||
|
const TestConfig = require("../../../tests/utilities/TestConfiguration") |
||||
|
|
||||
|
const syncApps = jest.fn() |
||||
|
const syncRows = jest.fn() |
||||
|
|
||||
|
jest.mock("../../usageQuotas/syncApps", () => ({ run: syncApps }) ) |
||||
|
jest.mock("../../usageQuotas/syncRows", () => ({ run: syncRows }) ) |
||||
|
|
||||
|
const migrations = require("../../usageQuotas") |
||||
|
|
||||
|
describe("run", () => { |
||||
|
let config = new TestConfig(false) |
||||
|
|
||||
|
beforeEach(async () => { |
||||
|
await config.init() |
||||
|
env._set("USE_QUOTAS", 1) |
||||
|
}) |
||||
|
|
||||
|
afterAll(config.end) |
||||
|
|
||||
|
it("runs the required migrations", async () => { |
||||
|
await migrations.run() |
||||
|
expect(syncApps).toHaveBeenCalledTimes(1) |
||||
|
expect(syncRows).toHaveBeenCalledTimes(1) |
||||
|
}) |
||||
|
}) |
||||
@ -0,0 +1,37 @@ |
|||||
|
const { getGlobalDB } = require("@budibase/backend-core/tenancy") |
||||
|
const TestConfig = require("../../../tests/utilities/TestConfiguration") |
||||
|
const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota") |
||||
|
const syncApps = require("../../usageQuotas/syncApps") |
||||
|
const env = require("../../../environment") |
||||
|
|
||||
|
describe("syncApps", () => { |
||||
|
let config = new TestConfig(false) |
||||
|
|
||||
|
beforeEach(async () => { |
||||
|
await config.init() |
||||
|
env._set("USE_QUOTAS", 1) |
||||
|
}) |
||||
|
|
||||
|
afterAll(config.end) |
||||
|
|
||||
|
it("runs successfully", async () => { |
||||
|
// create the usage quota doc and mock usages
|
||||
|
const db = getGlobalDB() |
||||
|
await getUsageQuotaDoc(db) |
||||
|
await update(Properties.APPS, 3) |
||||
|
|
||||
|
let usageDoc = await getUsageQuotaDoc(db) |
||||
|
expect(usageDoc.usageQuota.apps).toEqual(3) |
||||
|
|
||||
|
// create an extra app to test the migration
|
||||
|
await config.createApp("quota-test") |
||||
|
|
||||
|
// migrate
|
||||
|
await syncApps.run() |
||||
|
|
||||
|
// assert the migration worked
|
||||
|
usageDoc = await getUsageQuotaDoc(db) |
||||
|
expect(usageDoc.usageQuota.apps).toEqual(2) |
||||
|
}) |
||||
|
}) |
||||
|
|
||||
@ -0,0 +1,43 @@ |
|||||
|
const { getGlobalDB } = require("@budibase/backend-core/tenancy") |
||||
|
const TestConfig = require("../../../tests/utilities/TestConfiguration") |
||||
|
const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota") |
||||
|
const syncRows = require("../../usageQuotas/syncRows") |
||||
|
const env = require("../../../environment") |
||||
|
|
||||
|
describe("syncRows", () => { |
||||
|
let config = new TestConfig(false) |
||||
|
|
||||
|
beforeEach(async () => { |
||||
|
await config.init() |
||||
|
env._set("USE_QUOTAS", 1) |
||||
|
}) |
||||
|
|
||||
|
afterAll(config.end) |
||||
|
|
||||
|
it("runs successfully", async () => { |
||||
|
// create the usage quota doc and mock usages
|
||||
|
const db = getGlobalDB() |
||||
|
await getUsageQuotaDoc(db) |
||||
|
await update(Properties.ROW, 300) |
||||
|
|
||||
|
let usageDoc = await getUsageQuotaDoc(db) |
||||
|
expect(usageDoc.usageQuota.rows).toEqual(300) |
||||
|
|
||||
|
// app 1
|
||||
|
await config.createTable() |
||||
|
await config.createRow() |
||||
|
// app 2
|
||||
|
await config.createApp() |
||||
|
await config.createTable() |
||||
|
await config.createRow() |
||||
|
await config.createRow() |
||||
|
|
||||
|
// migrate
|
||||
|
await syncRows.run() |
||||
|
|
||||
|
// assert the migration worked
|
||||
|
usageDoc = await getUsageQuotaDoc(db) |
||||
|
expect(usageDoc.usageQuota.rows).toEqual(3) |
||||
|
}) |
||||
|
}) |
||||
|
|
||||
@ -0,0 +1,24 @@ |
|||||
|
const { |
||||
|
MIGRATIONS, |
||||
|
MIGRATION_DBS, |
||||
|
migrateIfRequired, |
||||
|
} = require("@budibase/backend-core/migrations") |
||||
|
const { useQuotas } = require("../../utilities/usageQuota") |
||||
|
const syncApps = require("./syncApps") |
||||
|
const syncRows = require("./syncRows") |
||||
|
|
||||
|
exports.run = async () => { |
||||
|
if (!useQuotas()) { |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
// Jan 2022
|
||||
|
await migrateIfRequired( |
||||
|
MIGRATION_DBS.GLOBAL_DB, |
||||
|
MIGRATIONS.QUOTAS_1, |
||||
|
async () => { |
||||
|
await syncApps.run() |
||||
|
await syncRows.run() |
||||
|
} |
||||
|
) |
||||
|
} |
||||
@ -0,0 +1,18 @@ |
|||||
|
const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy") |
||||
|
const { getAllApps } = require("@budibase/backend-core/db") |
||||
|
const CouchDB = require("../../db") |
||||
|
const { getUsageQuotaDoc } = require("../../utilities/usageQuota") |
||||
|
|
||||
|
exports.run = async () => { |
||||
|
const db = getGlobalDB() |
||||
|
// get app count
|
||||
|
const devApps = await getAllApps(CouchDB, { dev: true }) |
||||
|
const appCount = devApps ? devApps.length : 0 |
||||
|
|
||||
|
// sync app count
|
||||
|
const tenantId = getTenantId() |
||||
|
console.log(`[Tenant: ${tenantId}] Syncing app count: ${appCount}`) |
||||
|
const usageDoc = await getUsageQuotaDoc(db) |
||||
|
usageDoc.usageQuota.apps = appCount |
||||
|
await db.put(usageDoc) |
||||
|
} |
||||
@ -0,0 +1,21 @@ |
|||||
|
const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy") |
||||
|
const { getAllApps } = require("@budibase/backend-core/db") |
||||
|
const CouchDB = require("../../db") |
||||
|
const { getUsageQuotaDoc } = require("../../utilities/usageQuota") |
||||
|
const { getUniqueRows } = require("../../utilities/usageQuota/rows") |
||||
|
|
||||
|
exports.run = async () => { |
||||
|
const db = getGlobalDB() |
||||
|
// get all rows in all apps
|
||||
|
const allApps = await getAllApps(CouchDB, { all: true }) |
||||
|
const appIds = allApps ? allApps.map(app => app.appId) : [] |
||||
|
const rows = await getUniqueRows(appIds) |
||||
|
const rowCount = rows ? rows.length : 0 |
||||
|
|
||||
|
// sync row count
|
||||
|
const tenantId = getTenantId() |
||||
|
console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`) |
||||
|
const usageDoc = await getUsageQuotaDoc(db) |
||||
|
usageDoc.usageQuota.rows = rowCount |
||||
|
await db.put(usageDoc) |
||||
|
} |
||||
@ -0,0 +1,72 @@ |
|||||
|
const getTenantId = jest.fn() |
||||
|
jest.mock("@budibase/backend-core/tenancy", () => ({ |
||||
|
getTenantId |
||||
|
})) |
||||
|
const usageQuota = require("../../usageQuota") |
||||
|
const env = require("../../../environment") |
||||
|
|
||||
|
class TestConfiguration { |
||||
|
constructor() { |
||||
|
this.enableQuotas() |
||||
|
} |
||||
|
|
||||
|
enableQuotas = () => { |
||||
|
env.USE_QUOTAS = 1 |
||||
|
} |
||||
|
|
||||
|
disableQuotas = () => { |
||||
|
env.USE_QUOTAS = null |
||||
|
} |
||||
|
|
||||
|
setTenantId = (tenantId) => { |
||||
|
getTenantId.mockReturnValue(tenantId) |
||||
|
} |
||||
|
|
||||
|
setExcludedTenants = (tenants) => { |
||||
|
env.EXCLUDE_QUOTAS_TENANTS = tenants |
||||
|
} |
||||
|
|
||||
|
reset = () => { |
||||
|
this.disableQuotas() |
||||
|
this.setExcludedTenants(null) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
describe("usageQuota", () => { |
||||
|
let config |
||||
|
|
||||
|
beforeEach(() => { |
||||
|
config = new TestConfiguration() |
||||
|
}) |
||||
|
|
||||
|
afterEach(() => { |
||||
|
config.reset() |
||||
|
}) |
||||
|
|
||||
|
describe("useQuotas", () => { |
||||
|
it("works when no settings have been provided", () => { |
||||
|
config.reset() |
||||
|
expect(usageQuota.useQuotas()).toBe(false) |
||||
|
}) |
||||
|
it("honours USE_QUOTAS setting", () => { |
||||
|
config.disableQuotas() |
||||
|
expect(usageQuota.useQuotas()).toBe(false) |
||||
|
|
||||
|
config.enableQuotas() |
||||
|
expect(usageQuota.useQuotas()).toBe(true) |
||||
|
}) |
||||
|
it("honours EXCLUDE_QUOTAS_TENANTS setting", () => { |
||||
|
config.setTenantId("test") |
||||
|
|
||||
|
// tenantId is in the list
|
||||
|
config.setExcludedTenants("test, test2, test2") |
||||
|
expect(usageQuota.useQuotas()).toBe(false) |
||||
|
config.setExcludedTenants("test,test2,test2") |
||||
|
expect(usageQuota.useQuotas()).toBe(false) |
||||
|
|
||||
|
// tenantId is not in the list
|
||||
|
config.setTenantId("other") |
||||
|
expect(usageQuota.useQuotas()).toBe(true) |
||||
|
}) |
||||
|
}) |
||||
|
}) |
||||
@ -1,73 +0,0 @@ |
|||||
const env = require("../environment") |
|
||||
const { getGlobalDB } = require("@budibase/backend-core/tenancy") |
|
||||
const { |
|
||||
StaticDatabases, |
|
||||
generateNewUsageQuotaDoc, |
|
||||
} = require("@budibase/backend-core/db") |
|
||||
|
|
||||
function getNewQuotaReset() { |
|
||||
return Date.now() + 2592000000 |
|
||||
} |
|
||||
|
|
||||
exports.Properties = { |
|
||||
ROW: "rows", |
|
||||
UPLOAD: "storage", |
|
||||
VIEW: "views", |
|
||||
USER: "users", |
|
||||
AUTOMATION: "automationRuns", |
|
||||
APPS: "apps", |
|
||||
EMAILS: "emails", |
|
||||
} |
|
||||
|
|
||||
async function getUsageQuotaDoc(db) { |
|
||||
let quota |
|
||||
try { |
|
||||
quota = await db.get(StaticDatabases.PLATFORM_INFO.docs.usageQuota) |
|
||||
} catch (err) { |
|
||||
// doc doesn't exist. Create it
|
|
||||
quota = await db.post(generateNewUsageQuotaDoc()) |
|
||||
} |
|
||||
|
|
||||
return quota |
|
||||
} |
|
||||
|
|
||||
/** |
|
||||
* Given a specified tenantId this will add to the usage object for the specified property. |
|
||||
* @param {string} property The property which is to be added to (within the nested usageQuota object). |
|
||||
* @param {number} usage The amount (this can be negative) to adjust the number by. |
|
||||
* @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have |
|
||||
* also been reset after this call. |
|
||||
*/ |
|
||||
exports.update = async (property, usage) => { |
|
||||
if (!env.USE_QUOTAS) { |
|
||||
return |
|
||||
} |
|
||||
|
|
||||
try { |
|
||||
const db = getGlobalDB() |
|
||||
const quota = await getUsageQuotaDoc(db) |
|
||||
|
|
||||
// Check if the quota needs reset
|
|
||||
if (Date.now() >= quota.quotaReset) { |
|
||||
quota.quotaReset = getNewQuotaReset() |
|
||||
for (let prop of Object.keys(quota.usageQuota)) { |
|
||||
quota.usageQuota[prop] = 0 |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
// increment the quota
|
|
||||
quota.usageQuota[property] += usage |
|
||||
|
|
||||
if (quota.usageQuota[property] > quota.usageLimits[property]) { |
|
||||
throw new Error( |
|
||||
`You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.` |
|
||||
) |
|
||||
} |
|
||||
|
|
||||
// update the usage quotas
|
|
||||
await db.put(quota) |
|
||||
} catch (err) { |
|
||||
console.error(`Error updating usage quotas for ${property}`, err) |
|
||||
throw err |
|
||||
} |
|
||||
} |
|
||||
@ -0,0 +1,92 @@ |
|||||
|
const env = require("../../environment") |
||||
|
const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy") |
||||
|
const { |
||||
|
StaticDatabases, |
||||
|
generateNewUsageQuotaDoc, |
||||
|
} = require("@budibase/backend-core/db") |
||||
|
|
||||
|
exports.useQuotas = () => { |
||||
|
// check if quotas are enabled
|
||||
|
if (env.USE_QUOTAS) { |
||||
|
// check if there are any tenants without limits
|
||||
|
if (env.EXCLUDE_QUOTAS_TENANTS) { |
||||
|
const excludedTenants = env.EXCLUDE_QUOTAS_TENANTS.replace( |
||||
|
/\s/g, |
||||
|
"" |
||||
|
).split(",") |
||||
|
const tenantId = getTenantId() |
||||
|
if (excludedTenants.includes(tenantId)) { |
||||
|
return false |
||||
|
} |
||||
|
} |
||||
|
return true |
||||
|
} |
||||
|
return false |
||||
|
} |
||||
|
|
||||
|
exports.Properties = { |
||||
|
ROW: "rows", |
||||
|
UPLOAD: "storage", // doesn't work yet
|
||||
|
VIEW: "views", // doesn't work yet
|
||||
|
USER: "users", // doesn't work yet
|
||||
|
AUTOMATION: "automationRuns", // doesn't work yet
|
||||
|
APPS: "apps", |
||||
|
EMAILS: "emails", // doesn't work yet
|
||||
|
} |
||||
|
|
||||
|
exports.getUsageQuotaDoc = async db => { |
||||
|
let quota |
||||
|
try { |
||||
|
quota = await db.get(StaticDatabases.GLOBAL.docs.usageQuota) |
||||
|
} catch (err) { |
||||
|
// doc doesn't exist. Create it
|
||||
|
quota = generateNewUsageQuotaDoc() |
||||
|
const response = await db.put(quota) |
||||
|
quota._rev = response.rev |
||||
|
} |
||||
|
|
||||
|
return quota |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* Given a specified tenantId this will add to the usage object for the specified property. |
||||
|
* @param {string} property The property which is to be added to (within the nested usageQuota object). |
||||
|
* @param {number} usage The amount (this can be negative) to adjust the number by. |
||||
|
* @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have |
||||
|
* also been reset after this call. |
||||
|
*/ |
||||
|
exports.update = async (property, usage, opts = { dryRun: false }) => { |
||||
|
if (!exports.useQuotas()) { |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
try { |
||||
|
const db = getGlobalDB() |
||||
|
const quota = await exports.getUsageQuotaDoc(db) |
||||
|
|
||||
|
// increment the quota
|
||||
|
quota.usageQuota[property] += usage |
||||
|
|
||||
|
if ( |
||||
|
quota.usageQuota[property] > quota.usageLimits[property] && |
||||
|
usage > 0 // allow for decrementing usage when the quota is already exceeded
|
||||
|
) { |
||||
|
throw new Error( |
||||
|
`You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.` |
||||
|
) |
||||
|
} |
||||
|
|
||||
|
if (quota.usageQuota[property] < 0) { |
||||
|
// never go negative if the quota has previously been exceeded
|
||||
|
quota.usageQuota[property] = 0 |
||||
|
} |
||||
|
|
||||
|
// update the usage quotas
|
||||
|
if (!opts.dryRun) { |
||||
|
await db.put(quota) |
||||
|
} |
||||
|
} catch (err) { |
||||
|
console.error(`Error updating usage quotas for ${property}`, err) |
||||
|
throw err |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,74 @@ |
|||||
|
const { getRowParams, USER_METDATA_PREFIX } = require("../../db/utils") |
||||
|
const CouchDB = require("../../db") |
||||
|
const { isDevAppID, getDevelopmentAppID } = require("@budibase/backend-core/db") |
||||
|
|
||||
|
const ROW_EXCLUSIONS = [USER_METDATA_PREFIX] |
||||
|
|
||||
|
const getAppPairs = appIds => { |
||||
|
// collect the app ids into dev / prod pairs
|
||||
|
// keyed by the dev app id
|
||||
|
const pairs = {} |
||||
|
for (let appId of appIds) { |
||||
|
const devId = getDevelopmentAppID(appId) |
||||
|
if (!pairs[devId]) { |
||||
|
pairs[devId] = {} |
||||
|
} |
||||
|
if (isDevAppID(appId)) { |
||||
|
pairs[devId].devId = appId |
||||
|
} else { |
||||
|
pairs[devId].prodId = appId |
||||
|
} |
||||
|
} |
||||
|
return pairs |
||||
|
} |
||||
|
|
||||
|
const getAppRows = async appId => { |
||||
|
const appDb = new CouchDB(appId) |
||||
|
const response = await appDb.allDocs( |
||||
|
getRowParams(null, null, { |
||||
|
include_docs: false, |
||||
|
}) |
||||
|
) |
||||
|
return response.rows |
||||
|
.map(r => r.id) |
||||
|
.filter(id => { |
||||
|
for (let exclusion of ROW_EXCLUSIONS) { |
||||
|
if (id.startsWith(exclusion)) { |
||||
|
return false |
||||
|
} |
||||
|
} |
||||
|
return true |
||||
|
}) |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* Return a set of all rows in the given app ids. |
||||
|
* The returned rows will be unique on a per dev/prod app basis. |
||||
|
* Rows duplicates may exist across apps due to data import so they are not filtered out. |
||||
|
*/ |
||||
|
exports.getUniqueRows = async appIds => { |
||||
|
let uniqueRows = [] |
||||
|
const pairs = getAppPairs(appIds) |
||||
|
|
||||
|
for (let pair of Object.values(pairs)) { |
||||
|
let appRows = [] |
||||
|
for (let appId of [pair.devId, pair.prodId]) { |
||||
|
if (!appId) { |
||||
|
continue |
||||
|
} |
||||
|
try { |
||||
|
appRows.push(await getAppRows(appId)) |
||||
|
} catch (e) { |
||||
|
console.error(e) |
||||
|
// don't error out if we can't count the app rows, just continue
|
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// ensure uniqueness on a per app pair basis
|
||||
|
// this can't be done on all rows because app import results in
|
||||
|
// duplicate row ids across apps
|
||||
|
uniqueRows = uniqueRows.concat(...new Set(appRows)) |
||||
|
} |
||||
|
|
||||
|
return uniqueRows |
||||
|
} |
||||
@ -0,0 +1,18 @@ |
|||||
|
// UNUSED CODE
|
||||
|
// Preserved for future use
|
||||
|
|
||||
|
/* eslint-disable no-unused-vars */ |
||||
|
|
||||
|
function getNewQuotaReset() { |
||||
|
return Date.now() + 2592000000 |
||||
|
} |
||||
|
|
||||
|
function resetQuotasIfRequired(quota) { |
||||
|
// Check if the quota needs reset
|
||||
|
if (Date.now() >= quota.quotaReset) { |
||||
|
quota.quotaReset = getNewQuotaReset() |
||||
|
for (let prop of Object.keys(quota.usageQuota)) { |
||||
|
quota.usageQuota[prop] = 0 |
||||
|
} |
||||
|
} |
||||
|
} |
||||
Loading…
Reference in new issue