mirror of https://github.com/Budibase/budibase.git
Rory Powell
4 years ago
44 changed files with 459 additions and 278 deletions
@ -0,0 +1,40 @@ |
|||
import { |
|||
MigrationType, |
|||
MigrationName, |
|||
MigrationDefinition, |
|||
} from "@budibase/types" |
|||
|
|||
export const DEFINITIONS: MigrationDefinition[] = [ |
|||
{ |
|||
type: MigrationType.GLOBAL, |
|||
name: MigrationName.USER_EMAIL_VIEW_CASING, |
|||
}, |
|||
{ |
|||
type: MigrationType.GLOBAL, |
|||
name: MigrationName.QUOTAS_1, |
|||
}, |
|||
{ |
|||
type: MigrationType.APP, |
|||
name: MigrationName.APP_URLS, |
|||
}, |
|||
{ |
|||
type: MigrationType.GLOBAL, |
|||
name: MigrationName.DEVELOPER_QUOTA, |
|||
}, |
|||
{ |
|||
type: MigrationType.GLOBAL, |
|||
name: MigrationName.PUBLISHED_APP_QUOTA, |
|||
}, |
|||
{ |
|||
type: MigrationType.APP, |
|||
name: MigrationName.EVENT_APP_BACKFILL, |
|||
}, |
|||
{ |
|||
type: MigrationType.GLOBAL, |
|||
name: MigrationName.EVENT_GLOBAL_BACKFILL, |
|||
}, |
|||
{ |
|||
type: MigrationType.INSTALLATION, |
|||
name: MigrationName.EVENT_INSTALLATION_BACKFILL, |
|||
}, |
|||
] |
|||
@ -1,163 +0,0 @@ |
|||
const { DEFAULT_TENANT_ID } = require("../constants") |
|||
const { doWithDB } = require("../db") |
|||
const { DocumentTypes, StaticDatabases } = require("../db/constants") |
|||
const { getAllApps } = require("../db/utils") |
|||
const environment = require("../environment") |
|||
const { |
|||
doInTenant, |
|||
getTenantIds, |
|||
getGlobalDBName, |
|||
getTenantId, |
|||
} = require("../tenancy") |
|||
const context = require("../context") |
|||
|
|||
exports.MIGRATION_TYPES = { |
|||
GLOBAL: "global", // run once per tenant, recorded in global db, global db is provided as an argument
|
|||
APP: "app", // run per app, recorded in each app db, app db is provided as an argument
|
|||
INSTALLATION: "installation", // run once, recorded in global info db, global info db is provided as an argument
|
|||
} |
|||
|
|||
exports.getMigrationsDoc = async db => { |
|||
// get the migrations doc
|
|||
try { |
|||
return await db.get(DocumentTypes.MIGRATIONS) |
|||
} catch (err) { |
|||
if (err.status && err.status === 404) { |
|||
return { _id: DocumentTypes.MIGRATIONS } |
|||
} else { |
|||
console.error(err) |
|||
throw err |
|||
} |
|||
} |
|||
} |
|||
|
|||
exports.runMigration = async (migration, options = {}) => { |
|||
const migrationType = migration.type |
|||
let tenantId |
|||
if (migrationType !== exports.MIGRATION_TYPES.INSTALLATION) { |
|||
tenantId = getTenantId() |
|||
} |
|||
const migrationName = migration.name |
|||
const silent = migration.silent |
|||
|
|||
const log = message => { |
|||
if (!silent) { |
|||
console.log(message) |
|||
} |
|||
} |
|||
|
|||
// get the db to store the migration in
|
|||
let dbNames |
|||
if (migrationType === exports.MIGRATION_TYPES.GLOBAL) { |
|||
dbNames = [getGlobalDBName()] |
|||
} else if (migrationType === exports.MIGRATION_TYPES.APP) { |
|||
const apps = await getAllApps(migration.opts) |
|||
dbNames = apps.map(app => app.appId) |
|||
} else if (migrationType === exports.MIGRATION_TYPES.INSTALLATION) { |
|||
dbNames = [StaticDatabases.PLATFORM_INFO.name] |
|||
} else { |
|||
throw new Error(`Unrecognised migration type [${migrationType}]`) |
|||
} |
|||
|
|||
const length = dbNames.length |
|||
let count = 0 |
|||
|
|||
// run the migration against each db
|
|||
for (const dbName of dbNames) { |
|||
count++ |
|||
const lengthStatement = length > 1 ? `[${count}/${length}]` : "" |
|||
|
|||
await doWithDB(dbName, async db => { |
|||
try { |
|||
const doc = await exports.getMigrationsDoc(db) |
|||
|
|||
// exit if the migration has been performed already
|
|||
if (doc[migrationName]) { |
|||
if ( |
|||
options.force && |
|||
options.force[migrationType] && |
|||
options.force[migrationType].includes(migrationName) |
|||
) { |
|||
log( |
|||
`[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing` |
|||
) |
|||
} else { |
|||
// the migration has already been performed
|
|||
return |
|||
} |
|||
} |
|||
|
|||
log( |
|||
`[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}` |
|||
) |
|||
|
|||
if (migration.preventRetry) { |
|||
// eagerly set the completion date
|
|||
// so that we never run this migration twice even upon failure
|
|||
doc[migrationName] = Date.now() |
|||
const response = await db.put(doc) |
|||
doc._rev = response.rev |
|||
} |
|||
|
|||
// run the migration with tenant context
|
|||
if (migrationType === exports.MIGRATION_TYPES.APP) { |
|||
await context.doInAppContext(db.name, async () => { |
|||
await migration.fn(db) |
|||
}) |
|||
} else { |
|||
await migration.fn(db) |
|||
} |
|||
|
|||
log( |
|||
`[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete` |
|||
) |
|||
|
|||
// mark as complete
|
|||
doc[migrationName] = Date.now() |
|||
await db.put(doc) |
|||
} catch (err) { |
|||
console.error( |
|||
`[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `, |
|||
err |
|||
) |
|||
throw err |
|||
} |
|||
}) |
|||
} |
|||
} |
|||
|
|||
exports.runMigrations = async (migrations, options = {}) => { |
|||
let tenantIds |
|||
if (environment.MULTI_TENANCY) { |
|||
if (!options.tenantIds || !options.tenantIds.length) { |
|||
// run for all tenants
|
|||
tenantIds = await getTenantIds() |
|||
} else { |
|||
tenantIds = options.tenantIds |
|||
} |
|||
} else { |
|||
// single tenancy
|
|||
tenantIds = [DEFAULT_TENANT_ID] |
|||
} |
|||
|
|||
if (tenantIds.length > 1) { |
|||
console.log(`Checking migrations for ${tenantIds.length} tenants`) |
|||
} else { |
|||
console.log("Checking migrations") |
|||
} |
|||
|
|||
let count = 0 |
|||
// for all tenants
|
|||
for (const tenantId of tenantIds) { |
|||
count++ |
|||
if (tenantIds.length > 1) { |
|||
console.log(`Progress [${count}/${tenantIds.length}]`) |
|||
} |
|||
// for all migrations
|
|||
for (const migration of migrations) { |
|||
// run the migration
|
|||
await doInTenant(tenantId, () => exports.runMigration(migration, options)) |
|||
} |
|||
} |
|||
console.log("Migrations complete") |
|||
} |
|||
@ -0,0 +1,2 @@ |
|||
export * from "./migrations" |
|||
export * from "./definitions" |
|||
@ -0,0 +1,189 @@ |
|||
import { DEFAULT_TENANT_ID } from "../constants" |
|||
import { doWithDB } from "../db" |
|||
import { DocumentTypes, StaticDatabases } from "../db/constants" |
|||
import { getAllApps } from "../db/utils" |
|||
import environment from "../environment" |
|||
import { |
|||
doInTenant, |
|||
getTenantIds, |
|||
getGlobalDBName, |
|||
getTenantId, |
|||
} from "../tenancy" |
|||
import context from "../context" |
|||
import { DEFINITIONS } from "." |
|||
import { |
|||
Migration, |
|||
MigrationOptions, |
|||
MigrationType, |
|||
MigrationNoOpOptions, |
|||
} from "@budibase/types" |
|||
|
|||
export const getMigrationsDoc = async (db: any) => { |
|||
// get the migrations doc
|
|||
try { |
|||
return await db.get(DocumentTypes.MIGRATIONS) |
|||
} catch (err: any) { |
|||
if (err.status && err.status === 404) { |
|||
return { _id: DocumentTypes.MIGRATIONS } |
|||
} else { |
|||
console.error(err) |
|||
throw err |
|||
} |
|||
} |
|||
} |
|||
|
|||
export const backPopulateMigrations = async (opts: MigrationNoOpOptions) => { |
|||
// filter migrations to the type and populate a no-op migration
|
|||
const migrations: Migration[] = DEFINITIONS.filter( |
|||
def => def.type === opts.type |
|||
).map(d => ({ ...d, fn: () => {} })) |
|||
await runMigrations(migrations, { noOp: opts }) |
|||
} |
|||
|
|||
export const runMigration = async ( |
|||
migration: Migration, |
|||
options: MigrationOptions = {} |
|||
) => { |
|||
const migrationType = migration.type |
|||
let tenantId: string |
|||
if (migrationType !== MigrationType.INSTALLATION) { |
|||
tenantId = getTenantId() |
|||
} |
|||
const migrationName = migration.name |
|||
const silent = migration.silent |
|||
|
|||
const log = (message: string) => { |
|||
if (!silent) { |
|||
console.log(message) |
|||
} |
|||
} |
|||
|
|||
// get the db to store the migration in
|
|||
let dbNames |
|||
if (migrationType === MigrationType.GLOBAL) { |
|||
dbNames = [getGlobalDBName()] |
|||
} else if (migrationType === MigrationType.APP) { |
|||
if (options.noOp) { |
|||
dbNames = [options.noOp.appId] |
|||
} else { |
|||
const apps = await getAllApps(migration.appOpts) |
|||
dbNames = apps.map(app => app.appId) |
|||
} |
|||
} else if (migrationType === MigrationType.INSTALLATION) { |
|||
dbNames = [StaticDatabases.PLATFORM_INFO.name] |
|||
} else { |
|||
throw new Error(`Unrecognised migration type [${migrationType}]`) |
|||
} |
|||
|
|||
const length = dbNames.length |
|||
let count = 0 |
|||
|
|||
// run the migration against each db
|
|||
for (const dbName of dbNames) { |
|||
count++ |
|||
const lengthStatement = length > 1 ? `[${count}/${length}]` : "" |
|||
|
|||
await doWithDB(dbName, async (db: any) => { |
|||
try { |
|||
const doc = await exports.getMigrationsDoc(db) |
|||
|
|||
// the migration has already been run
|
|||
if (doc[migrationName]) { |
|||
// check for force
|
|||
if ( |
|||
options.force && |
|||
options.force[migrationType] && |
|||
options.force[migrationType].includes(migrationName) |
|||
) { |
|||
log( |
|||
`[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing` |
|||
) |
|||
} else { |
|||
// no force, exit
|
|||
return |
|||
} |
|||
} |
|||
|
|||
// check if the migration is not a no-op
|
|||
if (!options.noOp) { |
|||
log( |
|||
`[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}` |
|||
) |
|||
|
|||
if (migration.preventRetry) { |
|||
// eagerly set the completion date
|
|||
// so that we never run this migration twice even upon failure
|
|||
doc[migrationName] = Date.now() |
|||
const response = await db.put(doc) |
|||
doc._rev = response.rev |
|||
} |
|||
|
|||
// run the migration
|
|||
if (migrationType === MigrationType.APP) { |
|||
await context.doInAppContext(db.name, async () => { |
|||
await migration.fn(db) |
|||
}) |
|||
} else { |
|||
await migration.fn(db) |
|||
} |
|||
|
|||
log( |
|||
`[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete` |
|||
) |
|||
} |
|||
|
|||
// mark as complete
|
|||
doc[migrationName] = Date.now() |
|||
await db.put(doc) |
|||
} catch (err) { |
|||
console.error( |
|||
`[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `, |
|||
err |
|||
) |
|||
throw err |
|||
} |
|||
}) |
|||
} |
|||
} |
|||
|
|||
export const runMigrations = async ( |
|||
migrations: Migration[], |
|||
options: MigrationOptions = {} |
|||
) => { |
|||
let tenantIds |
|||
|
|||
if (environment.MULTI_TENANCY) { |
|||
if (options.noOp) { |
|||
tenantIds = [options.noOp.tenantId] |
|||
} else if (!options.tenantIds || !options.tenantIds.length) { |
|||
// run for all tenants
|
|||
tenantIds = await getTenantIds() |
|||
} else { |
|||
tenantIds = options.tenantIds |
|||
} |
|||
} else { |
|||
// single tenancy
|
|||
tenantIds = [DEFAULT_TENANT_ID] |
|||
} |
|||
|
|||
if (tenantIds.length > 1) { |
|||
console.log(`Checking migrations for ${tenantIds.length} tenants`) |
|||
} else { |
|||
console.log("Checking migrations") |
|||
} |
|||
|
|||
let count = 0 |
|||
// for all tenants
|
|||
for (const tenantId of tenantIds) { |
|||
count++ |
|||
if (tenantIds.length > 1) { |
|||
console.log(`Progress [${count}/${tenantIds.length}]`) |
|||
} |
|||
// for all migrations
|
|||
for (const migration of migrations) { |
|||
// run the migration
|
|||
await doInTenant(tenantId, () => runMigration(migration, options)) |
|||
} |
|||
} |
|||
console.log("Migrations complete") |
|||
} |
|||
@ -1,4 +0,0 @@ |
|||
module.exports = { |
|||
...require("../context"), |
|||
...require("./tenancy"), |
|||
} |
|||
@ -0,0 +1,9 @@ |
|||
import * as context from "../context" |
|||
import * as tenancy from "./tenancy" |
|||
|
|||
const pkg = { |
|||
...context, |
|||
...tenancy, |
|||
} |
|||
|
|||
export = pkg |
|||
@ -1,3 +0,0 @@ |
|||
export * from "./hosting" |
|||
export * from "./context" |
|||
export * from "./identification" |
|||
@ -1,4 +1,4 @@ |
|||
export * from "./documents" |
|||
export * from "./events" |
|||
export * from "./licensing" |
|||
export * from "./core" |
|||
export * from "./sdk/events" |
|||
export * from "./sdk/licensing" |
|||
export * from "./sdk" |
|||
|
|||
@ -1,5 +1,5 @@ |
|||
import { User, Account } from "../documents" |
|||
import { IdentityType } from "./identification" |
|||
import { IdentityType } from "./events/identification" |
|||
|
|||
export interface BaseContext { |
|||
_id: string |
|||
@ -1,4 +1,4 @@ |
|||
import { Hosting } from "." |
|||
import { Hosting } from ".." |
|||
|
|||
// GROUPS
|
|||
|
|||
@ -1,4 +1,4 @@ |
|||
import { ViewCalculation } from "../documents" |
|||
import { ViewCalculation } from "../../documents" |
|||
import { BaseEvent, TableExportFormat } from "./event" |
|||
|
|||
export interface ViewCreatedEvent extends BaseEvent { |
|||
@ -0,0 +1,5 @@ |
|||
export * from "./hosting" |
|||
export * from "./context" |
|||
export * from "./events" |
|||
export * from "./licensing" |
|||
export * from "./migrations" |
|||
@ -0,0 +1,54 @@ |
|||
export interface Migration extends MigrationDefinition { |
|||
appOpts?: object |
|||
fn: Function |
|||
silent?: boolean |
|||
preventRetry?: boolean |
|||
} |
|||
|
|||
export enum MigrationType { |
|||
// run once per tenant, recorded in global db, global db is provided as an argument
|
|||
GLOBAL = "global", |
|||
// run per app, recorded in each app db, app db is provided as an argument
|
|||
APP = "app", |
|||
// run once, recorded in global info db, global info db is provided as an argument
|
|||
INSTALLATION = "installation", |
|||
} |
|||
|
|||
export interface MigrationNoOpOptions { |
|||
type: MigrationType |
|||
tenantId: string |
|||
appId?: string |
|||
} |
|||
|
|||
/** |
|||
* e.g. |
|||
* { |
|||
* tenantIds: ['bb'], |
|||
* force: { |
|||
* global: ['quota_1'] |
|||
* } |
|||
* } |
|||
*/ |
|||
export interface MigrationOptions { |
|||
tenantIds?: string[] |
|||
force?: { |
|||
[type: string]: string[] |
|||
} |
|||
noOp?: MigrationNoOpOptions |
|||
} |
|||
|
|||
export enum MigrationName { |
|||
USER_EMAIL_VIEW_CASING = "user_email_view_casing", |
|||
QUOTAS_1 = "quotas_1", |
|||
APP_URLS = "app_urls", |
|||
DEVELOPER_QUOTA = "developer_quota", |
|||
PUBLISHED_APP_QUOTA = "published_apps_quota", |
|||
EVENT_APP_BACKFILL = "event_app_backfill", |
|||
EVENT_GLOBAL_BACKFILL = "event_global_backfill", |
|||
EVENT_INSTALLATION_BACKFILL = "event_installation_backfill", |
|||
} |
|||
|
|||
export interface MigrationDefinition { |
|||
type: MigrationType |
|||
name: MigrationName |
|||
} |
|||
Loading…
Reference in new issue