mirror of https://github.com/Budibase/budibase.git
Peter Clement
4 years ago
106 changed files with 5004 additions and 15655 deletions
@ -0,0 +1,17 @@ |
|||
export enum ContextKeys { |
|||
TENANT_ID = "tenantId", |
|||
GLOBAL_DB = "globalDb", |
|||
APP_ID = "appId", |
|||
IDENTITY = "identity", |
|||
// whatever the request app DB was
|
|||
CURRENT_DB = "currentDb", |
|||
// get the prod app DB from the request
|
|||
PROD_DB = "prodDb", |
|||
// get the dev app DB from the request
|
|||
DEV_DB = "devDb", |
|||
DB_OPTS = "dbOpts", |
|||
// check if something else is using the context, don't close DB
|
|||
TENANCY_IN_USE = "tenancyInUse", |
|||
APP_IN_USE = "appInUse", |
|||
IDENTITY_IN_USE = "identityInUse", |
|||
} |
|||
@ -1,354 +0,0 @@ |
|||
const env = require("../environment") |
|||
const { SEPARATOR, DocumentTypes } = require("../db/constants") |
|||
const { DEFAULT_TENANT_ID } = require("../constants") |
|||
const cls = require("./FunctionContext") |
|||
const { dangerousGetDB, closeDB } = require("../db") |
|||
const { getProdAppID, getDevelopmentAppID } = require("../db/conversions") |
|||
const { baseGlobalDBName } = require("../tenancy/utils") |
|||
const { isEqual } = require("lodash") |
|||
|
|||
// some test cases call functions directly, need to
|
|||
// store an app ID to pretend there is a context
|
|||
let TEST_APP_ID = null |
|||
|
|||
const ContextKeys = { |
|||
TENANT_ID: "tenantId", |
|||
GLOBAL_DB: "globalDb", |
|||
APP_ID: "appId", |
|||
IDENTITY: "identity", |
|||
// whatever the request app DB was
|
|||
CURRENT_DB: "currentDb", |
|||
// get the prod app DB from the request
|
|||
PROD_DB: "prodDb", |
|||
// get the dev app DB from the request
|
|||
DEV_DB: "devDb", |
|||
DB_OPTS: "dbOpts", |
|||
// check if something else is using the context, don't close DB
|
|||
IN_USE: "inUse", |
|||
} |
|||
|
|||
exports.DEFAULT_TENANT_ID = DEFAULT_TENANT_ID |
|||
|
|||
// this function makes sure the PouchDB objects are closed and
|
|||
// fully deleted when finished - this protects against memory leaks
|
|||
async function closeAppDBs() { |
|||
const dbKeys = [ |
|||
ContextKeys.CURRENT_DB, |
|||
ContextKeys.PROD_DB, |
|||
ContextKeys.DEV_DB, |
|||
] |
|||
for (let dbKey of dbKeys) { |
|||
const db = cls.getFromContext(dbKey) |
|||
if (!db) { |
|||
continue |
|||
} |
|||
await closeDB(db) |
|||
// clear the DB from context, incase someone tries to use it again
|
|||
cls.setOnContext(dbKey, null) |
|||
} |
|||
// clear the app ID now that the databases are closed
|
|||
if (cls.getFromContext(ContextKeys.APP_ID)) { |
|||
cls.setOnContext(ContextKeys.APP_ID, null) |
|||
} |
|||
if (cls.getFromContext(ContextKeys.DB_OPTS)) { |
|||
cls.setOnContext(ContextKeys.DB_OPTS, null) |
|||
} |
|||
} |
|||
|
|||
exports.closeTenancy = async () => { |
|||
if (env.USE_COUCH) { |
|||
await closeDB(exports.getGlobalDB()) |
|||
} |
|||
// clear from context now that database is closed/task is finished
|
|||
cls.setOnContext(ContextKeys.TENANT_ID, null) |
|||
cls.setOnContext(ContextKeys.GLOBAL_DB, null) |
|||
} |
|||
|
|||
exports.isDefaultTenant = () => { |
|||
return exports.getTenantId() === exports.DEFAULT_TENANT_ID |
|||
} |
|||
|
|||
exports.isMultiTenant = () => { |
|||
return env.MULTI_TENANCY |
|||
} |
|||
|
|||
// used for automations, API endpoints should always be in context already
|
|||
exports.doInTenant = (tenantId, task, { forceNew } = {}) => { |
|||
// the internal function is so that we can re-use an existing
|
|||
// context - don't want to close DB on a parent context
|
|||
async function internal(opts = { existing: false }) { |
|||
// set the tenant id
|
|||
if (!opts.existing) { |
|||
exports.updateTenantId(tenantId) |
|||
} |
|||
|
|||
try { |
|||
// invoke the task
|
|||
return await task() |
|||
} finally { |
|||
const using = cls.getFromContext(ContextKeys.IN_USE) |
|||
if (!using || using <= 1) { |
|||
await exports.closeTenancy() |
|||
} else { |
|||
cls.setOnContext(using - 1) |
|||
} |
|||
} |
|||
} |
|||
|
|||
const using = cls.getFromContext(ContextKeys.IN_USE) |
|||
if ( |
|||
!forceNew && |
|||
using && |
|||
cls.getFromContext(ContextKeys.TENANT_ID) === tenantId |
|||
) { |
|||
cls.setOnContext(ContextKeys.IN_USE, using + 1) |
|||
return internal({ existing: true }) |
|||
} else { |
|||
return cls.run(async () => { |
|||
cls.setOnContext(ContextKeys.IN_USE, 1) |
|||
return internal() |
|||
}) |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* Given an app ID this will attempt to retrieve the tenant ID from it. |
|||
* @return {null|string} The tenant ID found within the app ID. |
|||
*/ |
|||
exports.getTenantIDFromAppID = appId => { |
|||
if (!appId) { |
|||
return null |
|||
} |
|||
const split = appId.split(SEPARATOR) |
|||
const hasDev = split[1] === DocumentTypes.DEV |
|||
if ((hasDev && split.length === 3) || (!hasDev && split.length === 2)) { |
|||
return null |
|||
} |
|||
if (hasDev) { |
|||
return split[2] |
|||
} else { |
|||
return split[1] |
|||
} |
|||
} |
|||
|
|||
const setAppTenantId = appId => { |
|||
const appTenantId = |
|||
exports.getTenantIDFromAppID(appId) || exports.DEFAULT_TENANT_ID |
|||
exports.updateTenantId(appTenantId) |
|||
} |
|||
|
|||
exports.doInAppContext = (appId, task, { forceNew } = {}) => { |
|||
if (!appId) { |
|||
throw new Error("appId is required") |
|||
} |
|||
|
|||
const identity = exports.getIdentity() |
|||
|
|||
// the internal function is so that we can re-use an existing
|
|||
// context - don't want to close DB on a parent context
|
|||
async function internal(opts = { existing: false }) { |
|||
// set the app tenant id
|
|||
if (!opts.existing) { |
|||
setAppTenantId(appId) |
|||
} |
|||
// set the app ID
|
|||
cls.setOnContext(ContextKeys.APP_ID, appId) |
|||
// preserve the identity
|
|||
exports.setIdentity(identity) |
|||
try { |
|||
// invoke the task
|
|||
return await task() |
|||
} finally { |
|||
const using = cls.getFromContext(ContextKeys.IN_USE) |
|||
if (!using || using <= 1) { |
|||
await closeAppDBs() |
|||
} else { |
|||
cls.setOnContext(using - 1) |
|||
} |
|||
} |
|||
} |
|||
const using = cls.getFromContext(ContextKeys.IN_USE) |
|||
if (!forceNew && using && cls.getFromContext(ContextKeys.APP_ID) === appId) { |
|||
cls.setOnContext(ContextKeys.IN_USE, using + 1) |
|||
return internal({ existing: true }) |
|||
} else { |
|||
return cls.run(async () => { |
|||
cls.setOnContext(ContextKeys.IN_USE, 1) |
|||
return internal() |
|||
}) |
|||
} |
|||
} |
|||
|
|||
exports.doInIdentityContext = (identity, task) => { |
|||
if (!identity) { |
|||
throw new Error("identity is required") |
|||
} |
|||
|
|||
async function internal(opts = { existing: false }) { |
|||
if (!opts.existing) { |
|||
cls.setOnContext(ContextKeys.IDENTITY, identity) |
|||
// set the tenant so that doInTenant will preserve identity
|
|||
if (identity.tenantId) { |
|||
exports.updateTenantId(identity.tenantId) |
|||
} |
|||
} |
|||
|
|||
try { |
|||
// invoke the task
|
|||
return await task() |
|||
} finally { |
|||
const using = cls.getFromContext(ContextKeys.IN_USE) |
|||
if (!using || using <= 1) { |
|||
exports.setIdentity(null) |
|||
} else { |
|||
cls.setOnContext(using - 1) |
|||
} |
|||
} |
|||
} |
|||
|
|||
const existing = cls.getFromContext(ContextKeys.IDENTITY) |
|||
const using = cls.getFromContext(ContextKeys.IN_USE) |
|||
if (using && existing && existing._id === identity._id) { |
|||
cls.setOnContext(ContextKeys.IN_USE, using + 1) |
|||
return internal({ existing: true }) |
|||
} else { |
|||
return cls.run(async () => { |
|||
cls.setOnContext(ContextKeys.IN_USE, 1) |
|||
return internal({ existing: false }) |
|||
}) |
|||
} |
|||
} |
|||
|
|||
exports.setIdentity = identity => { |
|||
cls.setOnContext(ContextKeys.IDENTITY, identity) |
|||
} |
|||
|
|||
exports.getIdentity = () => { |
|||
try { |
|||
return cls.getFromContext(ContextKeys.IDENTITY) |
|||
} catch (e) { |
|||
// do nothing - identity is not in context
|
|||
} |
|||
} |
|||
|
|||
exports.updateTenantId = tenantId => { |
|||
cls.setOnContext(ContextKeys.TENANT_ID, tenantId) |
|||
if (env.USE_COUCH) { |
|||
exports.setGlobalDB(tenantId) |
|||
} |
|||
} |
|||
|
|||
exports.updateAppId = async appId => { |
|||
try { |
|||
// have to close first, before removing the databases from context
|
|||
await closeAppDBs() |
|||
cls.setOnContext(ContextKeys.APP_ID, appId) |
|||
} catch (err) { |
|||
if (env.isTest()) { |
|||
TEST_APP_ID = appId |
|||
} else { |
|||
throw err |
|||
} |
|||
} |
|||
} |
|||
|
|||
exports.setGlobalDB = tenantId => { |
|||
const dbName = baseGlobalDBName(tenantId) |
|||
const db = dangerousGetDB(dbName) |
|||
cls.setOnContext(ContextKeys.GLOBAL_DB, db) |
|||
return db |
|||
} |
|||
|
|||
exports.getGlobalDB = () => { |
|||
const db = cls.getFromContext(ContextKeys.GLOBAL_DB) |
|||
if (!db) { |
|||
throw new Error("Global DB not found") |
|||
} |
|||
return db |
|||
} |
|||
|
|||
exports.isTenantIdSet = () => { |
|||
const tenantId = cls.getFromContext(ContextKeys.TENANT_ID) |
|||
return !!tenantId |
|||
} |
|||
|
|||
exports.getTenantId = () => { |
|||
if (!exports.isMultiTenant()) { |
|||
return exports.DEFAULT_TENANT_ID |
|||
} |
|||
const tenantId = cls.getFromContext(ContextKeys.TENANT_ID) |
|||
if (!tenantId) { |
|||
throw new Error("Tenant id not found") |
|||
} |
|||
return tenantId |
|||
} |
|||
|
|||
exports.getAppId = () => { |
|||
const foundId = cls.getFromContext(ContextKeys.APP_ID) |
|||
if (!foundId && env.isTest() && TEST_APP_ID) { |
|||
return TEST_APP_ID |
|||
} else { |
|||
return foundId |
|||
} |
|||
} |
|||
|
|||
function getContextDB(key, opts) { |
|||
const dbOptsKey = `${key}${ContextKeys.DB_OPTS}` |
|||
let storedOpts = cls.getFromContext(dbOptsKey) |
|||
let db = cls.getFromContext(key) |
|||
if (db && isEqual(opts, storedOpts)) { |
|||
return db |
|||
} |
|||
|
|||
const appId = exports.getAppId() |
|||
let toUseAppId |
|||
|
|||
switch (key) { |
|||
case ContextKeys.CURRENT_DB: |
|||
toUseAppId = appId |
|||
break |
|||
case ContextKeys.PROD_DB: |
|||
toUseAppId = getProdAppID(appId) |
|||
break |
|||
case ContextKeys.DEV_DB: |
|||
toUseAppId = getDevelopmentAppID(appId) |
|||
break |
|||
} |
|||
|
|||
db = dangerousGetDB(toUseAppId, opts) |
|||
try { |
|||
cls.setOnContext(key, db) |
|||
if (opts) { |
|||
cls.setOnContext(dbOptsKey, opts) |
|||
} |
|||
} catch (err) { |
|||
if (!env.isTest()) { |
|||
throw err |
|||
} |
|||
} |
|||
return db |
|||
} |
|||
|
|||
/** |
|||
* Opens the app database based on whatever the request |
|||
* contained, dev or prod. |
|||
*/ |
|||
exports.getAppDB = (opts = null) => { |
|||
return getContextDB(ContextKeys.CURRENT_DB, opts) |
|||
} |
|||
|
|||
/** |
|||
* This specifically gets the prod app ID, if the request |
|||
* contained a development app ID, this will open the prod one. |
|||
*/ |
|||
exports.getProdAppDB = (opts = null) => { |
|||
return getContextDB(ContextKeys.PROD_DB, opts) |
|||
} |
|||
|
|||
/** |
|||
* This specifically gets the dev app ID, if the request |
|||
* contained a prod app ID, this will open the dev one. |
|||
*/ |
|||
exports.getDevAppDB = (opts = null) => { |
|||
return getContextDB(ContextKeys.DEV_DB, opts) |
|||
} |
|||
@ -0,0 +1,247 @@ |
|||
import env from "../environment" |
|||
import { SEPARATOR, DocumentTypes } from "../db/constants" |
|||
import cls from "./FunctionContext" |
|||
import { dangerousGetDB, closeDB } from "../db" |
|||
import { baseGlobalDBName } from "../tenancy/utils" |
|||
import { IdentityContext } from "@budibase/types" |
|||
import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants" |
|||
import { ContextKeys } from "./constants" |
|||
import { |
|||
updateUsing, |
|||
closeWithUsing, |
|||
setAppTenantId, |
|||
setIdentity, |
|||
closeAppDBs, |
|||
getContextDB, |
|||
} from "./utils" |
|||
|
|||
export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID |
|||
|
|||
// some test cases call functions directly, need to
|
|||
// store an app ID to pretend there is a context
|
|||
let TEST_APP_ID: string | null = null |
|||
|
|||
export const closeTenancy = async () => { |
|||
let db |
|||
try { |
|||
if (env.USE_COUCH) { |
|||
db = getGlobalDB() |
|||
} |
|||
} catch (err) { |
|||
// no DB found - skip closing
|
|||
return |
|||
} |
|||
await closeDB(db) |
|||
// clear from context now that database is closed/task is finished
|
|||
cls.setOnContext(ContextKeys.TENANT_ID, null) |
|||
cls.setOnContext(ContextKeys.GLOBAL_DB, null) |
|||
} |
|||
|
|||
// export const isDefaultTenant = () => {
|
|||
// return getTenantId() === DEFAULT_TENANT_ID
|
|||
// }
|
|||
|
|||
export const isMultiTenant = () => { |
|||
return env.MULTI_TENANCY |
|||
} |
|||
|
|||
/** |
|||
* Given an app ID this will attempt to retrieve the tenant ID from it. |
|||
* @return {null|string} The tenant ID found within the app ID. |
|||
*/ |
|||
export const getTenantIDFromAppID = (appId: string) => { |
|||
if (!appId) { |
|||
return null |
|||
} |
|||
const split = appId.split(SEPARATOR) |
|||
const hasDev = split[1] === DocumentTypes.DEV |
|||
if ((hasDev && split.length === 3) || (!hasDev && split.length === 2)) { |
|||
return null |
|||
} |
|||
if (hasDev) { |
|||
return split[2] |
|||
} else { |
|||
return split[1] |
|||
} |
|||
} |
|||
|
|||
// used for automations, API endpoints should always be in context already
|
|||
export const doInTenant = (tenantId: string | null, task: any) => { |
|||
// the internal function is so that we can re-use an existing
|
|||
// context - don't want to close DB on a parent context
|
|||
async function internal(opts = { existing: false }) { |
|||
// set the tenant id + global db if this is a new context
|
|||
if (!opts.existing) { |
|||
updateTenantId(tenantId) |
|||
} |
|||
|
|||
try { |
|||
// invoke the task
|
|||
return await task() |
|||
} finally { |
|||
await closeWithUsing(ContextKeys.TENANCY_IN_USE, () => { |
|||
return closeTenancy() |
|||
}) |
|||
} |
|||
} |
|||
|
|||
const existing = cls.getFromContext(ContextKeys.TENANT_ID) === tenantId |
|||
return updateUsing(ContextKeys.TENANCY_IN_USE, existing, internal) |
|||
} |
|||
|
|||
export const doInAppContext = (appId: string, task: any) => { |
|||
if (!appId) { |
|||
throw new Error("appId is required") |
|||
} |
|||
|
|||
const identity = getIdentity() |
|||
|
|||
// the internal function is so that we can re-use an existing
|
|||
// context - don't want to close DB on a parent context
|
|||
async function internal(opts = { existing: false }) { |
|||
// set the app tenant id
|
|||
if (!opts.existing) { |
|||
setAppTenantId(appId) |
|||
} |
|||
// set the app ID
|
|||
cls.setOnContext(ContextKeys.APP_ID, appId) |
|||
|
|||
// preserve the identity
|
|||
if (identity) { |
|||
setIdentity(identity) |
|||
} |
|||
try { |
|||
// invoke the task
|
|||
return await task() |
|||
} finally { |
|||
await closeWithUsing(ContextKeys.APP_IN_USE, async () => { |
|||
await closeAppDBs() |
|||
await closeTenancy() |
|||
}) |
|||
} |
|||
} |
|||
const existing = cls.getFromContext(ContextKeys.APP_ID) === appId |
|||
return updateUsing(ContextKeys.APP_IN_USE, existing, internal) |
|||
} |
|||
|
|||
export const doInIdentityContext = (identity: IdentityContext, task: any) => { |
|||
if (!identity) { |
|||
throw new Error("identity is required") |
|||
} |
|||
|
|||
async function internal(opts = { existing: false }) { |
|||
if (!opts.existing) { |
|||
cls.setOnContext(ContextKeys.IDENTITY, identity) |
|||
// set the tenant so that doInTenant will preserve identity
|
|||
if (identity.tenantId) { |
|||
updateTenantId(identity.tenantId) |
|||
} |
|||
} |
|||
|
|||
try { |
|||
// invoke the task
|
|||
return await task() |
|||
} finally { |
|||
await closeWithUsing(ContextKeys.IDENTITY_IN_USE, async () => { |
|||
setIdentity(null) |
|||
await closeTenancy() |
|||
}) |
|||
} |
|||
} |
|||
|
|||
const existing = cls.getFromContext(ContextKeys.IDENTITY) |
|||
return updateUsing(ContextKeys.IDENTITY_IN_USE, existing, internal) |
|||
} |
|||
|
|||
export const getIdentity = (): IdentityContext | undefined => { |
|||
try { |
|||
return cls.getFromContext(ContextKeys.IDENTITY) |
|||
} catch (e) { |
|||
// do nothing - identity is not in context
|
|||
} |
|||
} |
|||
|
|||
export const updateTenantId = (tenantId: string | null) => { |
|||
cls.setOnContext(ContextKeys.TENANT_ID, tenantId) |
|||
if (env.USE_COUCH) { |
|||
setGlobalDB(tenantId) |
|||
} |
|||
} |
|||
|
|||
export const updateAppId = async (appId: string) => { |
|||
try { |
|||
// have to close first, before removing the databases from context
|
|||
await closeAppDBs() |
|||
cls.setOnContext(ContextKeys.APP_ID, appId) |
|||
} catch (err) { |
|||
if (env.isTest()) { |
|||
TEST_APP_ID = appId |
|||
} else { |
|||
throw err |
|||
} |
|||
} |
|||
} |
|||
|
|||
export const setGlobalDB = (tenantId: string | null) => { |
|||
const dbName = baseGlobalDBName(tenantId) |
|||
const db = dangerousGetDB(dbName) |
|||
cls.setOnContext(ContextKeys.GLOBAL_DB, db) |
|||
return db |
|||
} |
|||
|
|||
export const getGlobalDB = () => { |
|||
const db = cls.getFromContext(ContextKeys.GLOBAL_DB) |
|||
if (!db) { |
|||
throw new Error("Global DB not found") |
|||
} |
|||
return db |
|||
} |
|||
|
|||
export const isTenantIdSet = () => { |
|||
const tenantId = cls.getFromContext(ContextKeys.TENANT_ID) |
|||
return !!tenantId |
|||
} |
|||
|
|||
export const getTenantId = () => { |
|||
if (!isMultiTenant()) { |
|||
return DEFAULT_TENANT_ID |
|||
} |
|||
const tenantId = cls.getFromContext(ContextKeys.TENANT_ID) |
|||
if (!tenantId) { |
|||
throw new Error("Tenant id not found") |
|||
} |
|||
return tenantId |
|||
} |
|||
|
|||
export const getAppId = () => { |
|||
const foundId = cls.getFromContext(ContextKeys.APP_ID) |
|||
if (!foundId && env.isTest() && TEST_APP_ID) { |
|||
return TEST_APP_ID |
|||
} else { |
|||
return foundId |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* Opens the app database based on whatever the request |
|||
* contained, dev or prod. |
|||
*/ |
|||
export const getAppDB = (opts?: any) => { |
|||
return getContextDB(ContextKeys.CURRENT_DB, opts) |
|||
} |
|||
|
|||
/** |
|||
* This specifically gets the prod app ID, if the request |
|||
* contained a development app ID, this will open the prod one. |
|||
*/ |
|||
export const getProdAppDB = (opts?: any) => { |
|||
return getContextDB(ContextKeys.PROD_DB, opts) |
|||
} |
|||
|
|||
/** |
|||
* This specifically gets the dev app ID, if the request |
|||
* contained a prod app ID, this will open the dev one. |
|||
*/ |
|||
export const getDevAppDB = (opts?: any) => { |
|||
return getContextDB(ContextKeys.DEV_DB, opts) |
|||
} |
|||
@ -0,0 +1,148 @@ |
|||
import "../../../tests/utilities/TestConfiguration" |
|||
import * as context from ".." |
|||
import { DEFAULT_TENANT_ID } from "../../constants" |
|||
import env from "../../environment" |
|||
|
|||
// must use require to spy index file exports due to known issue in jest
|
|||
const dbUtils = require("../../db") |
|||
jest.spyOn(dbUtils, "closeDB") |
|||
jest.spyOn(dbUtils, "dangerousGetDB") |
|||
|
|||
describe("context", () => { |
|||
beforeEach(() => { |
|||
jest.clearAllMocks() |
|||
}) |
|||
|
|||
describe("doInTenant", () => { |
|||
describe("single-tenancy", () => { |
|||
it("defaults to the default tenant", () => { |
|||
const tenantId = context.getTenantId() |
|||
expect(tenantId).toBe(DEFAULT_TENANT_ID) |
|||
}) |
|||
|
|||
it("defaults to the default tenant db", async () => { |
|||
await context.doInTenant(DEFAULT_TENANT_ID, () => { |
|||
const db = context.getGlobalDB() |
|||
expect(db.name).toBe("global-db") |
|||
}) |
|||
expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1) |
|||
expect(dbUtils.closeDB).toHaveBeenCalledTimes(1) |
|||
}) |
|||
}) |
|||
|
|||
describe("multi-tenancy", () => { |
|||
beforeEach(() => { |
|||
env._set("MULTI_TENANCY", 1) |
|||
}) |
|||
|
|||
it("fails when no tenant id is set", () => { |
|||
const test = () => { |
|||
let error |
|||
try { |
|||
context.getTenantId() |
|||
} catch (e: any) { |
|||
error = e |
|||
} |
|||
expect(error.message).toBe("Tenant id not found") |
|||
} |
|||
|
|||
// test under no tenancy
|
|||
test() |
|||
|
|||
// test after tenancy has been accessed to ensure cleanup
|
|||
context.doInTenant("test", () => {}) |
|||
test() |
|||
}) |
|||
|
|||
it("fails when no tenant db is set", () => { |
|||
const test = () => { |
|||
let error |
|||
try { |
|||
context.getGlobalDB() |
|||
} catch (e: any) { |
|||
error = e |
|||
} |
|||
expect(error.message).toBe("Global DB not found") |
|||
} |
|||
|
|||
// test under no tenancy
|
|||
test() |
|||
|
|||
// test after tenancy has been accessed to ensure cleanup
|
|||
context.doInTenant("test", () => {}) |
|||
test() |
|||
}) |
|||
|
|||
it("sets tenant id", () => { |
|||
context.doInTenant("test", () => { |
|||
const tenantId = context.getTenantId() |
|||
expect(tenantId).toBe("test") |
|||
}) |
|||
}) |
|||
|
|||
it("initialises the tenant db", async () => { |
|||
await context.doInTenant("test", () => { |
|||
const db = context.getGlobalDB() |
|||
expect(db.name).toBe("test_global-db") |
|||
}) |
|||
expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1) |
|||
expect(dbUtils.closeDB).toHaveBeenCalledTimes(1) |
|||
}) |
|||
|
|||
it("sets the tenant id when nested with same tenant id", async () => { |
|||
await context.doInTenant("test", async () => { |
|||
const tenantId = context.getTenantId() |
|||
expect(tenantId).toBe("test") |
|||
|
|||
await context.doInTenant("test", async () => { |
|||
const tenantId = context.getTenantId() |
|||
expect(tenantId).toBe("test") |
|||
|
|||
await context.doInTenant("test", () => { |
|||
const tenantId = context.getTenantId() |
|||
expect(tenantId).toBe("test") |
|||
}) |
|||
}) |
|||
}) |
|||
}) |
|||
|
|||
it("initialises the tenant db when nested with same tenant id", async () => { |
|||
await context.doInTenant("test", async () => { |
|||
const db = context.getGlobalDB() |
|||
expect(db.name).toBe("test_global-db") |
|||
|
|||
await context.doInTenant("test", async () => { |
|||
const db = context.getGlobalDB() |
|||
expect(db.name).toBe("test_global-db") |
|||
|
|||
await context.doInTenant("test", () => { |
|||
const db = context.getGlobalDB() |
|||
expect(db.name).toBe("test_global-db") |
|||
}) |
|||
}) |
|||
}) |
|||
|
|||
// only 1 db is opened and closed
|
|||
expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1) |
|||
expect(dbUtils.closeDB).toHaveBeenCalledTimes(1) |
|||
}) |
|||
|
|||
it("sets different tenant id inside another context", () => { |
|||
context.doInTenant("test", () => { |
|||
const tenantId = context.getTenantId() |
|||
expect(tenantId).toBe("test") |
|||
|
|||
context.doInTenant("nested", () => { |
|||
const tenantId = context.getTenantId() |
|||
expect(tenantId).toBe("nested") |
|||
|
|||
context.doInTenant("double-nested", () => { |
|||
const tenantId = context.getTenantId() |
|||
expect(tenantId).toBe("double-nested") |
|||
}) |
|||
}) |
|||
}) |
|||
}) |
|||
}) |
|||
}) |
|||
}) |
|||
@ -0,0 +1,113 @@ |
|||
import { |
|||
DEFAULT_TENANT_ID, |
|||
getAppId, |
|||
getTenantIDFromAppID, |
|||
updateTenantId, |
|||
} from "./index" |
|||
import cls from "./FunctionContext" |
|||
import { IdentityContext } from "@budibase/types" |
|||
import { ContextKeys } from "./constants" |
|||
import { dangerousGetDB, closeDB } from "../db" |
|||
import { isEqual } from "lodash" |
|||
import { getDevelopmentAppID, getProdAppID } from "../db/conversions" |
|||
import env from "../environment" |
|||
|
|||
export async function updateUsing( |
|||
usingKey: string, |
|||
existing: boolean, |
|||
internal: (opts: { existing: boolean }) => Promise<any> |
|||
) { |
|||
const using = cls.getFromContext(usingKey) |
|||
if (using && existing) { |
|||
cls.setOnContext(usingKey, using + 1) |
|||
return internal({ existing: true }) |
|||
} else { |
|||
return cls.run(async () => { |
|||
cls.setOnContext(usingKey, 1) |
|||
return internal({ existing: false }) |
|||
}) |
|||
} |
|||
} |
|||
|
|||
export async function closeWithUsing( |
|||
usingKey: string, |
|||
closeFn: () => Promise<any> |
|||
) { |
|||
const using = cls.getFromContext(usingKey) |
|||
if (!using || using <= 1) { |
|||
await closeFn() |
|||
} else { |
|||
cls.setOnContext(usingKey, using - 1) |
|||
} |
|||
} |
|||
|
|||
export const setAppTenantId = (appId: string) => { |
|||
const appTenantId = getTenantIDFromAppID(appId) || DEFAULT_TENANT_ID |
|||
updateTenantId(appTenantId) |
|||
} |
|||
|
|||
export const setIdentity = (identity: IdentityContext | null) => { |
|||
cls.setOnContext(ContextKeys.IDENTITY, identity) |
|||
} |
|||
|
|||
// this function makes sure the PouchDB objects are closed and
|
|||
// fully deleted when finished - this protects against memory leaks
|
|||
export async function closeAppDBs() { |
|||
const dbKeys = [ |
|||
ContextKeys.CURRENT_DB, |
|||
ContextKeys.PROD_DB, |
|||
ContextKeys.DEV_DB, |
|||
] |
|||
for (let dbKey of dbKeys) { |
|||
const db = cls.getFromContext(dbKey) |
|||
if (!db) { |
|||
continue |
|||
} |
|||
await closeDB(db) |
|||
// clear the DB from context, incase someone tries to use it again
|
|||
cls.setOnContext(dbKey, null) |
|||
} |
|||
// clear the app ID now that the databases are closed
|
|||
if (cls.getFromContext(ContextKeys.APP_ID)) { |
|||
cls.setOnContext(ContextKeys.APP_ID, null) |
|||
} |
|||
if (cls.getFromContext(ContextKeys.DB_OPTS)) { |
|||
cls.setOnContext(ContextKeys.DB_OPTS, null) |
|||
} |
|||
} |
|||
|
|||
export function getContextDB(key: string, opts: any) { |
|||
const dbOptsKey = `${key}${ContextKeys.DB_OPTS}` |
|||
let storedOpts = cls.getFromContext(dbOptsKey) |
|||
let db = cls.getFromContext(key) |
|||
if (db && isEqual(opts, storedOpts)) { |
|||
return db |
|||
} |
|||
|
|||
const appId = getAppId() |
|||
let toUseAppId |
|||
|
|||
switch (key) { |
|||
case ContextKeys.CURRENT_DB: |
|||
toUseAppId = appId |
|||
break |
|||
case ContextKeys.PROD_DB: |
|||
toUseAppId = getProdAppID(appId) |
|||
break |
|||
case ContextKeys.DEV_DB: |
|||
toUseAppId = getDevelopmentAppID(appId) |
|||
break |
|||
} |
|||
db = dangerousGetDB(toUseAppId, opts) |
|||
try { |
|||
cls.setOnContext(key, db) |
|||
if (opts) { |
|||
cls.setOnContext(dbOptsKey, opts) |
|||
} |
|||
} catch (err) { |
|||
if (!env.isTest()) { |
|||
throw err |
|||
} |
|||
} |
|||
return db |
|||
} |
|||
File diff suppressed because it is too large
@ -0,0 +1,28 @@ |
|||
export const TriggerStepID = { |
|||
ROW_SAVED: "ROW_SAVED", |
|||
ROW_UPDATED: "ROW_UPDATED", |
|||
ROW_DELETED: "ROW_DELETED", |
|||
WEBHOOK: "WEBHOOK", |
|||
APP: "APP", |
|||
CRON: "CRON", |
|||
} |
|||
|
|||
export const ActionStepID = { |
|||
SEND_EMAIL_SMTP: "SEND_EMAIL_SMTP", |
|||
CREATE_ROW: "CREATE_ROW", |
|||
UPDATE_ROW: "UPDATE_ROW", |
|||
DELETE_ROW: "DELETE_ROW", |
|||
OUTGOING_WEBHOOK: "OUTGOING_WEBHOOK", |
|||
EXECUTE_SCRIPT: "EXECUTE_SCRIPT", |
|||
EXECUTE_QUERY: "EXECUTE_QUERY", |
|||
SERVER_LOG: "SERVER_LOG", |
|||
DELAY: "DELAY", |
|||
FILTER: "FILTER", |
|||
QUERY_ROWS: "QUERY_ROWS", |
|||
LOOP: "LOOP", |
|||
// these used to be lowercase step IDs, maintain for backwards compat
|
|||
discord: "discord", |
|||
slack: "slack", |
|||
zapier: "zapier", |
|||
integromat: "integromat", |
|||
} |
|||
@ -0,0 +1,121 @@ |
|||
const Command = require("../structures/Command") |
|||
const { CommandWords } = require("../constants") |
|||
const fs = require("fs") |
|||
const { join } = require("path") |
|||
const { getAllDbs } = require("../core/db") |
|||
const tar = require("tar") |
|||
const { progressBar } = require("../utils") |
|||
const { |
|||
TEMP_DIR, |
|||
COUCH_DIR, |
|||
MINIO_DIR, |
|||
getConfig, |
|||
replication, |
|||
getPouches, |
|||
} = require("./utils") |
|||
const { exportObjects, importObjects } = require("./objectStore") |
|||
|
|||
async function exportBackup(opts) { |
|||
const envFile = opts.env || undefined |
|||
let filename = opts["export"] || opts |
|||
if (typeof filename !== "string") { |
|||
filename = `backup-${new Date().toISOString()}.tar.gz` |
|||
} |
|||
const config = await getConfig(envFile) |
|||
const dbList = await getAllDbs(config["COUCH_DB_URL"]) |
|||
const { Remote, Local } = getPouches(config) |
|||
if (fs.existsSync(TEMP_DIR)) { |
|||
fs.rmSync(TEMP_DIR, { recursive: true }) |
|||
} |
|||
const couchDir = join(TEMP_DIR, COUCH_DIR) |
|||
fs.mkdirSync(TEMP_DIR) |
|||
fs.mkdirSync(couchDir) |
|||
console.log("CouchDB Export") |
|||
const bar = progressBar(dbList.length) |
|||
let count = 0 |
|||
for (let db of dbList) { |
|||
bar.update(++count) |
|||
const remote = new Remote(db) |
|||
const local = new Local(join(TEMP_DIR, COUCH_DIR, db)) |
|||
await replication(remote, local) |
|||
} |
|||
bar.stop() |
|||
console.log("S3 Export") |
|||
await exportObjects() |
|||
tar.create( |
|||
{ |
|||
sync: true, |
|||
gzip: true, |
|||
file: filename, |
|||
cwd: join(TEMP_DIR), |
|||
}, |
|||
[COUCH_DIR, MINIO_DIR] |
|||
) |
|||
fs.rmSync(TEMP_DIR, { recursive: true }) |
|||
console.log(`Generated export file - ${filename}`) |
|||
} |
|||
|
|||
async function importBackup(opts) { |
|||
const envFile = opts.env || undefined |
|||
const filename = opts["import"] || opts |
|||
const config = await getConfig(envFile) |
|||
if (!filename || !fs.existsSync(filename)) { |
|||
console.error("Cannot import without specifying a valid file to import") |
|||
process.exit(-1) |
|||
} |
|||
if (fs.existsSync(TEMP_DIR)) { |
|||
fs.rmSync(TEMP_DIR, { recursive: true }) |
|||
} |
|||
fs.mkdirSync(TEMP_DIR) |
|||
tar.extract({ |
|||
sync: true, |
|||
cwd: join(TEMP_DIR), |
|||
file: filename, |
|||
}) |
|||
const { Remote, Local } = getPouches(config) |
|||
const dbList = fs.readdirSync(join(TEMP_DIR, COUCH_DIR)) |
|||
console.log("CouchDB Import") |
|||
const bar = progressBar(dbList.length) |
|||
let count = 0 |
|||
for (let db of dbList) { |
|||
bar.update(++count) |
|||
const remote = new Remote(db) |
|||
const local = new Local(join(TEMP_DIR, COUCH_DIR, db)) |
|||
await replication(local, remote) |
|||
} |
|||
bar.stop() |
|||
console.log("MinIO Import") |
|||
await importObjects() |
|||
console.log("Import complete") |
|||
fs.rmSync(TEMP_DIR, { recursive: true }) |
|||
} |
|||
|
|||
async function pickOne(opts) { |
|||
if (opts["import"]) { |
|||
return importBackup(opts) |
|||
} else if (opts["export"]) { |
|||
return exportBackup(opts) |
|||
} |
|||
} |
|||
|
|||
const command = new Command(`${CommandWords.BACKUPS}`) |
|||
.addHelp( |
|||
"Allows building backups of Budibase, as well as importing a backup to a new instance." |
|||
) |
|||
.addSubOption( |
|||
"--export [filename]", |
|||
"Export a backup from an existing Budibase installation.", |
|||
exportBackup |
|||
) |
|||
.addSubOption( |
|||
"--import [filename]", |
|||
"Import a backup to a new Budibase installation.", |
|||
importBackup |
|||
) |
|||
.addSubOption( |
|||
"--env [envFile]", |
|||
"Provide an environment variable file to configure the CLI.", |
|||
pickOne |
|||
) |
|||
|
|||
exports.command = command |
|||
@ -0,0 +1,63 @@ |
|||
const { |
|||
ObjectStoreBuckets, |
|||
ObjectStore, |
|||
retrieve, |
|||
uploadDirectory, |
|||
makeSureBucketExists, |
|||
} = require("@budibase/backend-core/objectStore") |
|||
const fs = require("fs") |
|||
const { join } = require("path") |
|||
const { TEMP_DIR, MINIO_DIR } = require("./utils") |
|||
const { progressBar } = require("../utils") |
|||
|
|||
const bucketList = Object.values(ObjectStoreBuckets) |
|||
|
|||
exports.exportObjects = async () => { |
|||
const path = join(TEMP_DIR, MINIO_DIR) |
|||
fs.mkdirSync(path) |
|||
let fullList = [] |
|||
for (let bucket of bucketList) { |
|||
const client = ObjectStore(bucket) |
|||
try { |
|||
await client.headBucket().promise() |
|||
} catch (err) { |
|||
continue |
|||
} |
|||
const list = await client.listObjectsV2().promise() |
|||
fullList = fullList.concat(list.Contents.map(el => ({ ...el, bucket }))) |
|||
} |
|||
const bar = progressBar(fullList.length) |
|||
let count = 0 |
|||
for (let object of fullList) { |
|||
const filename = object.Key |
|||
const data = await retrieve(object.bucket, filename) |
|||
const possiblePath = filename.split("/") |
|||
if (possiblePath.length > 1) { |
|||
const dirs = possiblePath.slice(0, possiblePath.length - 1) |
|||
fs.mkdirSync(join(path, object.bucket, ...dirs), { recursive: true }) |
|||
} |
|||
fs.writeFileSync(join(path, object.bucket, ...possiblePath), data) |
|||
bar.update(++count) |
|||
} |
|||
bar.stop() |
|||
} |
|||
|
|||
exports.importObjects = async () => { |
|||
const path = join(TEMP_DIR, MINIO_DIR) |
|||
const buckets = fs.readdirSync(path) |
|||
let total = 0 |
|||
buckets.forEach(bucket => { |
|||
const files = fs.readdirSync(join(path, bucket)) |
|||
total += files.length |
|||
}) |
|||
const bar = progressBar(total) |
|||
let count = 0 |
|||
for (let bucket of buckets) { |
|||
const client = ObjectStore(bucket) |
|||
await makeSureBucketExists(client, bucket) |
|||
const files = await uploadDirectory(bucket, join(path, bucket), "/") |
|||
count += files.length |
|||
bar.update(count) |
|||
} |
|||
bar.stop() |
|||
} |
|||
@ -0,0 +1,88 @@ |
|||
const dotenv = require("dotenv") |
|||
const fs = require("fs") |
|||
const { string } = require("../questions") |
|||
const { getPouch } = require("../core/db") |
|||
|
|||
exports.DEFAULT_COUCH = "http://budibase:budibase@localhost:10000/db/" |
|||
exports.DEFAULT_MINIO = "http://localhost:10000/" |
|||
exports.TEMP_DIR = ".temp" |
|||
exports.COUCH_DIR = "couchdb" |
|||
exports.MINIO_DIR = "minio" |
|||
|
|||
const REQUIRED = [ |
|||
{ value: "MAIN_PORT", default: "10000" }, |
|||
{ value: "COUCH_DB_URL", default: exports.DEFAULT_COUCH }, |
|||
{ value: "MINIO_URL", default: exports.DEFAULT_MINIO }, |
|||
{ value: "MINIO_ACCESS_KEY" }, |
|||
{ value: "MINIO_SECRET_KEY" }, |
|||
] |
|||
|
|||
exports.checkURLs = config => { |
|||
const mainPort = config["MAIN_PORT"], |
|||
username = config["COUCH_DB_USER"], |
|||
password = config["COUCH_DB_PASSWORD"] |
|||
if (!config["COUCH_DB_URL"] && mainPort && username && password) { |
|||
config[ |
|||
"COUCH_DB_URL" |
|||
] = `http://${username}:${password}@localhost:${mainPort}/db/` |
|||
} |
|||
if (!config["MINIO_URL"]) { |
|||
config["MINIO_URL"] = exports.DEFAULT_MINIO |
|||
} |
|||
return config |
|||
} |
|||
|
|||
exports.askQuestions = async () => { |
|||
console.log( |
|||
"*** NOTE: use a .env file to load these parameters repeatedly ***" |
|||
) |
|||
let config = {} |
|||
for (let property of REQUIRED) { |
|||
config[property.value] = await string(property.value, property.default) |
|||
} |
|||
return config |
|||
} |
|||
|
|||
exports.loadEnvironment = path => { |
|||
if (!fs.existsSync(path)) { |
|||
throw "Unable to file specified .env file" |
|||
} |
|||
const env = fs.readFileSync(path, "utf8") |
|||
const config = exports.checkURLs(dotenv.parse(env)) |
|||
for (let required of REQUIRED) { |
|||
if (!config[required.value]) { |
|||
throw `Cannot find "${required.value}" property in .env file` |
|||
} |
|||
} |
|||
return config |
|||
} |
|||
|
|||
// true is the default value passed by commander
|
|||
exports.getConfig = async (envFile = true) => { |
|||
let config |
|||
if (envFile !== true) { |
|||
config = exports.loadEnvironment(envFile) |
|||
} else { |
|||
config = await exports.askQuestions() |
|||
} |
|||
return config |
|||
} |
|||
|
|||
exports.replication = (from, to) => { |
|||
return new Promise((resolve, reject) => { |
|||
from.replicate |
|||
.to(to) |
|||
.on("complete", () => { |
|||
resolve() |
|||
}) |
|||
.on("error", err => { |
|||
reject(err) |
|||
}) |
|||
}) |
|||
} |
|||
|
|||
exports.getPouches = config => { |
|||
const Remote = getPouch(config["COUCH_DB_URL"]) |
|||
const Local = getPouch() |
|||
return { Remote, Local } |
|||
} |
|||
@ -0,0 +1,38 @@ |
|||
const PouchDB = require("pouchdb") |
|||
const { checkSlashesInUrl } = require("../utils") |
|||
const fetch = require("node-fetch") |
|||
|
|||
/** |
|||
* Fully qualified URL including username and password, or nothing for local |
|||
*/ |
|||
exports.getPouch = (url = undefined) => { |
|||
let POUCH_DB_DEFAULTS = {} |
|||
if (!url) { |
|||
POUCH_DB_DEFAULTS = { |
|||
prefix: undefined, |
|||
adapter: "leveldb", |
|||
} |
|||
} else { |
|||
POUCH_DB_DEFAULTS = { |
|||
prefix: url, |
|||
} |
|||
} |
|||
const replicationStream = require("pouchdb-replication-stream") |
|||
PouchDB.plugin(replicationStream.plugin) |
|||
PouchDB.adapter("writableStream", replicationStream.adapters.writableStream) |
|||
return PouchDB.defaults(POUCH_DB_DEFAULTS) |
|||
} |
|||
|
|||
exports.getAllDbs = async url => { |
|||
const response = await fetch( |
|||
checkSlashesInUrl(encodeURI(`${url}/_all_dbs`)), |
|||
{ |
|||
method: "GET", |
|||
} |
|||
) |
|||
if (response.status === 200) { |
|||
return await response.json() |
|||
} else { |
|||
throw "Cannot connect to CouchDB instance" |
|||
} |
|||
} |
|||
@ -1,6 +1,7 @@ |
|||
const analytics = require("./analytics") |
|||
const hosting = require("./hosting") |
|||
const backups = require("./backups") |
|||
|
|||
exports.getCommands = () => { |
|||
return [hosting.command, analytics.command] |
|||
return [hosting.command, analytics.command, backups.command] |
|||
} |
|||
|
|||
@ -0,0 +1,34 @@ |
|||
const os = require("os") |
|||
const { join } = require("path") |
|||
const fs = require("fs") |
|||
const PREBUILDS = "prebuilds" |
|||
const ARCH = `${os.platform()}-${os.arch()}` |
|||
const PREBUILD_DIR = join(process.execPath, "..", PREBUILDS, ARCH) |
|||
|
|||
checkForBinaries() |
|||
|
|||
function checkForBinaries() { |
|||
const readDir = join(__filename, "..", "..", PREBUILDS, ARCH) |
|||
if (fs.existsSync(PREBUILD_DIR) || !fs.existsSync(readDir)) { |
|||
return |
|||
} |
|||
const natives = fs.readdirSync(readDir) |
|||
if (fs.existsSync(readDir)) { |
|||
fs.mkdirSync(PREBUILD_DIR, { recursive: true }) |
|||
for (let native of natives) { |
|||
const filename = `${native.split(".fake")[0]}.node` |
|||
fs.cpSync(join(readDir, native), join(PREBUILD_DIR, filename)) |
|||
} |
|||
} |
|||
} |
|||
|
|||
function cleanup() { |
|||
if (fs.existsSync(PREBUILD_DIR)) { |
|||
fs.rmSync(PREBUILD_DIR, { recursive: true }) |
|||
} |
|||
} |
|||
|
|||
const events = ["exit", "SIGINT", "SIGUSR1", "SIGUSR2", "uncaughtException"] |
|||
events.forEach(event => { |
|||
process.on(event, cleanup) |
|||
}) |
|||
File diff suppressed because it is too large
File diff suppressed because it is too large
File diff suppressed because it is too large
Some files were not shown because too many files changed in this diff
Loading…
Reference in new issue