mirror of https://github.com/Budibase/budibase.git
Rory Powell
4 years ago
44 changed files with 459 additions and 278 deletions
@ -0,0 +1,40 @@ |
|||||
|
import { |
||||
|
MigrationType, |
||||
|
MigrationName, |
||||
|
MigrationDefinition, |
||||
|
} from "@budibase/types" |
||||
|
|
||||
|
export const DEFINITIONS: MigrationDefinition[] = [ |
||||
|
{ |
||||
|
type: MigrationType.GLOBAL, |
||||
|
name: MigrationName.USER_EMAIL_VIEW_CASING, |
||||
|
}, |
||||
|
{ |
||||
|
type: MigrationType.GLOBAL, |
||||
|
name: MigrationName.QUOTAS_1, |
||||
|
}, |
||||
|
{ |
||||
|
type: MigrationType.APP, |
||||
|
name: MigrationName.APP_URLS, |
||||
|
}, |
||||
|
{ |
||||
|
type: MigrationType.GLOBAL, |
||||
|
name: MigrationName.DEVELOPER_QUOTA, |
||||
|
}, |
||||
|
{ |
||||
|
type: MigrationType.GLOBAL, |
||||
|
name: MigrationName.PUBLISHED_APP_QUOTA, |
||||
|
}, |
||||
|
{ |
||||
|
type: MigrationType.APP, |
||||
|
name: MigrationName.EVENT_APP_BACKFILL, |
||||
|
}, |
||||
|
{ |
||||
|
type: MigrationType.GLOBAL, |
||||
|
name: MigrationName.EVENT_GLOBAL_BACKFILL, |
||||
|
}, |
||||
|
{ |
||||
|
type: MigrationType.INSTALLATION, |
||||
|
name: MigrationName.EVENT_INSTALLATION_BACKFILL, |
||||
|
}, |
||||
|
] |
||||
@ -1,163 +0,0 @@ |
|||||
const { DEFAULT_TENANT_ID } = require("../constants") |
|
||||
const { doWithDB } = require("../db") |
|
||||
const { DocumentTypes, StaticDatabases } = require("../db/constants") |
|
||||
const { getAllApps } = require("../db/utils") |
|
||||
const environment = require("../environment") |
|
||||
const { |
|
||||
doInTenant, |
|
||||
getTenantIds, |
|
||||
getGlobalDBName, |
|
||||
getTenantId, |
|
||||
} = require("../tenancy") |
|
||||
const context = require("../context") |
|
||||
|
|
||||
exports.MIGRATION_TYPES = { |
|
||||
GLOBAL: "global", // run once per tenant, recorded in global db, global db is provided as an argument
|
|
||||
APP: "app", // run per app, recorded in each app db, app db is provided as an argument
|
|
||||
INSTALLATION: "installation", // run once, recorded in global info db, global info db is provided as an argument
|
|
||||
} |
|
||||
|
|
||||
exports.getMigrationsDoc = async db => { |
|
||||
// get the migrations doc
|
|
||||
try { |
|
||||
return await db.get(DocumentTypes.MIGRATIONS) |
|
||||
} catch (err) { |
|
||||
if (err.status && err.status === 404) { |
|
||||
return { _id: DocumentTypes.MIGRATIONS } |
|
||||
} else { |
|
||||
console.error(err) |
|
||||
throw err |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
exports.runMigration = async (migration, options = {}) => { |
|
||||
const migrationType = migration.type |
|
||||
let tenantId |
|
||||
if (migrationType !== exports.MIGRATION_TYPES.INSTALLATION) { |
|
||||
tenantId = getTenantId() |
|
||||
} |
|
||||
const migrationName = migration.name |
|
||||
const silent = migration.silent |
|
||||
|
|
||||
const log = message => { |
|
||||
if (!silent) { |
|
||||
console.log(message) |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
// get the db to store the migration in
|
|
||||
let dbNames |
|
||||
if (migrationType === exports.MIGRATION_TYPES.GLOBAL) { |
|
||||
dbNames = [getGlobalDBName()] |
|
||||
} else if (migrationType === exports.MIGRATION_TYPES.APP) { |
|
||||
const apps = await getAllApps(migration.opts) |
|
||||
dbNames = apps.map(app => app.appId) |
|
||||
} else if (migrationType === exports.MIGRATION_TYPES.INSTALLATION) { |
|
||||
dbNames = [StaticDatabases.PLATFORM_INFO.name] |
|
||||
} else { |
|
||||
throw new Error(`Unrecognised migration type [${migrationType}]`) |
|
||||
} |
|
||||
|
|
||||
const length = dbNames.length |
|
||||
let count = 0 |
|
||||
|
|
||||
// run the migration against each db
|
|
||||
for (const dbName of dbNames) { |
|
||||
count++ |
|
||||
const lengthStatement = length > 1 ? `[${count}/${length}]` : "" |
|
||||
|
|
||||
await doWithDB(dbName, async db => { |
|
||||
try { |
|
||||
const doc = await exports.getMigrationsDoc(db) |
|
||||
|
|
||||
// exit if the migration has been performed already
|
|
||||
if (doc[migrationName]) { |
|
||||
if ( |
|
||||
options.force && |
|
||||
options.force[migrationType] && |
|
||||
options.force[migrationType].includes(migrationName) |
|
||||
) { |
|
||||
log( |
|
||||
`[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing` |
|
||||
) |
|
||||
} else { |
|
||||
// the migration has already been performed
|
|
||||
return |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
log( |
|
||||
`[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}` |
|
||||
) |
|
||||
|
|
||||
if (migration.preventRetry) { |
|
||||
// eagerly set the completion date
|
|
||||
// so that we never run this migration twice even upon failure
|
|
||||
doc[migrationName] = Date.now() |
|
||||
const response = await db.put(doc) |
|
||||
doc._rev = response.rev |
|
||||
} |
|
||||
|
|
||||
// run the migration with tenant context
|
|
||||
if (migrationType === exports.MIGRATION_TYPES.APP) { |
|
||||
await context.doInAppContext(db.name, async () => { |
|
||||
await migration.fn(db) |
|
||||
}) |
|
||||
} else { |
|
||||
await migration.fn(db) |
|
||||
} |
|
||||
|
|
||||
log( |
|
||||
`[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete` |
|
||||
) |
|
||||
|
|
||||
// mark as complete
|
|
||||
doc[migrationName] = Date.now() |
|
||||
await db.put(doc) |
|
||||
} catch (err) { |
|
||||
console.error( |
|
||||
`[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `, |
|
||||
err |
|
||||
) |
|
||||
throw err |
|
||||
} |
|
||||
}) |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
exports.runMigrations = async (migrations, options = {}) => { |
|
||||
let tenantIds |
|
||||
if (environment.MULTI_TENANCY) { |
|
||||
if (!options.tenantIds || !options.tenantIds.length) { |
|
||||
// run for all tenants
|
|
||||
tenantIds = await getTenantIds() |
|
||||
} else { |
|
||||
tenantIds = options.tenantIds |
|
||||
} |
|
||||
} else { |
|
||||
// single tenancy
|
|
||||
tenantIds = [DEFAULT_TENANT_ID] |
|
||||
} |
|
||||
|
|
||||
if (tenantIds.length > 1) { |
|
||||
console.log(`Checking migrations for ${tenantIds.length} tenants`) |
|
||||
} else { |
|
||||
console.log("Checking migrations") |
|
||||
} |
|
||||
|
|
||||
let count = 0 |
|
||||
// for all tenants
|
|
||||
for (const tenantId of tenantIds) { |
|
||||
count++ |
|
||||
if (tenantIds.length > 1) { |
|
||||
console.log(`Progress [${count}/${tenantIds.length}]`) |
|
||||
} |
|
||||
// for all migrations
|
|
||||
for (const migration of migrations) { |
|
||||
// run the migration
|
|
||||
await doInTenant(tenantId, () => exports.runMigration(migration, options)) |
|
||||
} |
|
||||
} |
|
||||
console.log("Migrations complete") |
|
||||
} |
|
||||
@ -0,0 +1,2 @@ |
|||||
|
export * from "./migrations" |
||||
|
export * from "./definitions" |
||||
@ -0,0 +1,189 @@ |
|||||
|
import { DEFAULT_TENANT_ID } from "../constants" |
||||
|
import { doWithDB } from "../db" |
||||
|
import { DocumentTypes, StaticDatabases } from "../db/constants" |
||||
|
import { getAllApps } from "../db/utils" |
||||
|
import environment from "../environment" |
||||
|
import { |
||||
|
doInTenant, |
||||
|
getTenantIds, |
||||
|
getGlobalDBName, |
||||
|
getTenantId, |
||||
|
} from "../tenancy" |
||||
|
import context from "../context" |
||||
|
import { DEFINITIONS } from "." |
||||
|
import { |
||||
|
Migration, |
||||
|
MigrationOptions, |
||||
|
MigrationType, |
||||
|
MigrationNoOpOptions, |
||||
|
} from "@budibase/types" |
||||
|
|
||||
|
export const getMigrationsDoc = async (db: any) => { |
||||
|
// get the migrations doc
|
||||
|
try { |
||||
|
return await db.get(DocumentTypes.MIGRATIONS) |
||||
|
} catch (err: any) { |
||||
|
if (err.status && err.status === 404) { |
||||
|
return { _id: DocumentTypes.MIGRATIONS } |
||||
|
} else { |
||||
|
console.error(err) |
||||
|
throw err |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
export const backPopulateMigrations = async (opts: MigrationNoOpOptions) => { |
||||
|
// filter migrations to the type and populate a no-op migration
|
||||
|
const migrations: Migration[] = DEFINITIONS.filter( |
||||
|
def => def.type === opts.type |
||||
|
).map(d => ({ ...d, fn: () => {} })) |
||||
|
await runMigrations(migrations, { noOp: opts }) |
||||
|
} |
||||
|
|
||||
|
export const runMigration = async ( |
||||
|
migration: Migration, |
||||
|
options: MigrationOptions = {} |
||||
|
) => { |
||||
|
const migrationType = migration.type |
||||
|
let tenantId: string |
||||
|
if (migrationType !== MigrationType.INSTALLATION) { |
||||
|
tenantId = getTenantId() |
||||
|
} |
||||
|
const migrationName = migration.name |
||||
|
const silent = migration.silent |
||||
|
|
||||
|
const log = (message: string) => { |
||||
|
if (!silent) { |
||||
|
console.log(message) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// get the db to store the migration in
|
||||
|
let dbNames |
||||
|
if (migrationType === MigrationType.GLOBAL) { |
||||
|
dbNames = [getGlobalDBName()] |
||||
|
} else if (migrationType === MigrationType.APP) { |
||||
|
if (options.noOp) { |
||||
|
dbNames = [options.noOp.appId] |
||||
|
} else { |
||||
|
const apps = await getAllApps(migration.appOpts) |
||||
|
dbNames = apps.map(app => app.appId) |
||||
|
} |
||||
|
} else if (migrationType === MigrationType.INSTALLATION) { |
||||
|
dbNames = [StaticDatabases.PLATFORM_INFO.name] |
||||
|
} else { |
||||
|
throw new Error(`Unrecognised migration type [${migrationType}]`) |
||||
|
} |
||||
|
|
||||
|
const length = dbNames.length |
||||
|
let count = 0 |
||||
|
|
||||
|
// run the migration against each db
|
||||
|
for (const dbName of dbNames) { |
||||
|
count++ |
||||
|
const lengthStatement = length > 1 ? `[${count}/${length}]` : "" |
||||
|
|
||||
|
await doWithDB(dbName, async (db: any) => { |
||||
|
try { |
||||
|
const doc = await exports.getMigrationsDoc(db) |
||||
|
|
||||
|
// the migration has already been run
|
||||
|
if (doc[migrationName]) { |
||||
|
// check for force
|
||||
|
if ( |
||||
|
options.force && |
||||
|
options.force[migrationType] && |
||||
|
options.force[migrationType].includes(migrationName) |
||||
|
) { |
||||
|
log( |
||||
|
`[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing` |
||||
|
) |
||||
|
} else { |
||||
|
// no force, exit
|
||||
|
return |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// check if the migration is not a no-op
|
||||
|
if (!options.noOp) { |
||||
|
log( |
||||
|
`[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}` |
||||
|
) |
||||
|
|
||||
|
if (migration.preventRetry) { |
||||
|
// eagerly set the completion date
|
||||
|
// so that we never run this migration twice even upon failure
|
||||
|
doc[migrationName] = Date.now() |
||||
|
const response = await db.put(doc) |
||||
|
doc._rev = response.rev |
||||
|
} |
||||
|
|
||||
|
// run the migration
|
||||
|
if (migrationType === MigrationType.APP) { |
||||
|
await context.doInAppContext(db.name, async () => { |
||||
|
await migration.fn(db) |
||||
|
}) |
||||
|
} else { |
||||
|
await migration.fn(db) |
||||
|
} |
||||
|
|
||||
|
log( |
||||
|
`[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete` |
||||
|
) |
||||
|
} |
||||
|
|
||||
|
// mark as complete
|
||||
|
doc[migrationName] = Date.now() |
||||
|
await db.put(doc) |
||||
|
} catch (err) { |
||||
|
console.error( |
||||
|
`[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `, |
||||
|
err |
||||
|
) |
||||
|
throw err |
||||
|
} |
||||
|
}) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
export const runMigrations = async ( |
||||
|
migrations: Migration[], |
||||
|
options: MigrationOptions = {} |
||||
|
) => { |
||||
|
let tenantIds |
||||
|
|
||||
|
if (environment.MULTI_TENANCY) { |
||||
|
if (options.noOp) { |
||||
|
tenantIds = [options.noOp.tenantId] |
||||
|
} else if (!options.tenantIds || !options.tenantIds.length) { |
||||
|
// run for all tenants
|
||||
|
tenantIds = await getTenantIds() |
||||
|
} else { |
||||
|
tenantIds = options.tenantIds |
||||
|
} |
||||
|
} else { |
||||
|
// single tenancy
|
||||
|
tenantIds = [DEFAULT_TENANT_ID] |
||||
|
} |
||||
|
|
||||
|
if (tenantIds.length > 1) { |
||||
|
console.log(`Checking migrations for ${tenantIds.length} tenants`) |
||||
|
} else { |
||||
|
console.log("Checking migrations") |
||||
|
} |
||||
|
|
||||
|
let count = 0 |
||||
|
// for all tenants
|
||||
|
for (const tenantId of tenantIds) { |
||||
|
count++ |
||||
|
if (tenantIds.length > 1) { |
||||
|
console.log(`Progress [${count}/${tenantIds.length}]`) |
||||
|
} |
||||
|
// for all migrations
|
||||
|
for (const migration of migrations) { |
||||
|
// run the migration
|
||||
|
await doInTenant(tenantId, () => runMigration(migration, options)) |
||||
|
} |
||||
|
} |
||||
|
console.log("Migrations complete") |
||||
|
} |
||||
@ -1,4 +0,0 @@ |
|||||
module.exports = { |
|
||||
...require("../context"), |
|
||||
...require("./tenancy"), |
|
||||
} |
|
||||
@ -0,0 +1,9 @@ |
|||||
|
import * as context from "../context" |
||||
|
import * as tenancy from "./tenancy" |
||||
|
|
||||
|
const pkg = { |
||||
|
...context, |
||||
|
...tenancy, |
||||
|
} |
||||
|
|
||||
|
export = pkg |
||||
@ -1,3 +0,0 @@ |
|||||
export * from "./hosting" |
|
||||
export * from "./context" |
|
||||
export * from "./identification" |
|
||||
@ -1,4 +1,4 @@ |
|||||
export * from "./documents" |
export * from "./documents" |
||||
export * from "./events" |
export * from "./sdk/events" |
||||
export * from "./licensing" |
export * from "./sdk/licensing" |
||||
export * from "./core" |
export * from "./sdk" |
||||
|
|||||
@ -1,5 +1,5 @@ |
|||||
import { User, Account } from "../documents" |
import { User, Account } from "../documents" |
||||
import { IdentityType } from "./identification" |
import { IdentityType } from "./events/identification" |
||||
|
|
||||
export interface BaseContext { |
export interface BaseContext { |
||||
_id: string |
_id: string |
||||
@ -1,4 +1,4 @@ |
|||||
import { Hosting } from "." |
import { Hosting } from ".." |
||||
|
|
||||
// GROUPS
|
// GROUPS
|
||||
|
|
||||
@ -1,4 +1,4 @@ |
|||||
import { ViewCalculation } from "../documents" |
import { ViewCalculation } from "../../documents" |
||||
import { BaseEvent, TableExportFormat } from "./event" |
import { BaseEvent, TableExportFormat } from "./event" |
||||
|
|
||||
export interface ViewCreatedEvent extends BaseEvent { |
export interface ViewCreatedEvent extends BaseEvent { |
||||
@ -0,0 +1,5 @@ |
|||||
|
export * from "./hosting" |
||||
|
export * from "./context" |
||||
|
export * from "./events" |
||||
|
export * from "./licensing" |
||||
|
export * from "./migrations" |
||||
@ -0,0 +1,54 @@ |
|||||
|
export interface Migration extends MigrationDefinition { |
||||
|
appOpts?: object |
||||
|
fn: Function |
||||
|
silent?: boolean |
||||
|
preventRetry?: boolean |
||||
|
} |
||||
|
|
||||
|
export enum MigrationType { |
||||
|
// run once per tenant, recorded in global db, global db is provided as an argument
|
||||
|
GLOBAL = "global", |
||||
|
// run per app, recorded in each app db, app db is provided as an argument
|
||||
|
APP = "app", |
||||
|
// run once, recorded in global info db, global info db is provided as an argument
|
||||
|
INSTALLATION = "installation", |
||||
|
} |
||||
|
|
||||
|
export interface MigrationNoOpOptions { |
||||
|
type: MigrationType |
||||
|
tenantId: string |
||||
|
appId?: string |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* e.g. |
||||
|
* { |
||||
|
* tenantIds: ['bb'], |
||||
|
* force: { |
||||
|
* global: ['quota_1'] |
||||
|
* } |
||||
|
* } |
||||
|
*/ |
||||
|
export interface MigrationOptions { |
||||
|
tenantIds?: string[] |
||||
|
force?: { |
||||
|
[type: string]: string[] |
||||
|
} |
||||
|
noOp?: MigrationNoOpOptions |
||||
|
} |
||||
|
|
||||
|
export enum MigrationName { |
||||
|
USER_EMAIL_VIEW_CASING = "user_email_view_casing", |
||||
|
QUOTAS_1 = "quotas_1", |
||||
|
APP_URLS = "app_urls", |
||||
|
DEVELOPER_QUOTA = "developer_quota", |
||||
|
PUBLISHED_APP_QUOTA = "published_apps_quota", |
||||
|
EVENT_APP_BACKFILL = "event_app_backfill", |
||||
|
EVENT_GLOBAL_BACKFILL = "event_global_backfill", |
||||
|
EVENT_INSTALLATION_BACKFILL = "event_installation_backfill", |
||||
|
} |
||||
|
|
||||
|
export interface MigrationDefinition { |
||||
|
type: MigrationType |
||||
|
name: MigrationName |
||||
|
} |
||||
Loading…
Reference in new issue