forked from tsai/budibase
Michael Drury
4 years ago
committed by
GitHub
GitHub
21 changed files with 1514 additions and 3440 deletions
File diff suppressed because it is too large
@ -0,0 +1,121 @@ |
|||
const Command = require("../structures/Command") |
|||
const { CommandWords } = require("../constants") |
|||
const fs = require("fs") |
|||
const { join } = require("path") |
|||
const { getAllDbs } = require("../core/db") |
|||
const tar = require("tar") |
|||
const { progressBar } = require("../utils") |
|||
const { |
|||
TEMP_DIR, |
|||
COUCH_DIR, |
|||
MINIO_DIR, |
|||
getConfig, |
|||
replication, |
|||
getPouches, |
|||
} = require("./utils") |
|||
const { exportObjects, importObjects } = require("./objectStore") |
|||
|
|||
async function exportBackup(opts) { |
|||
const envFile = opts.env || undefined |
|||
let filename = opts["export"] || opts |
|||
if (typeof filename !== "string") { |
|||
filename = `backup-${new Date().toISOString()}.tar.gz` |
|||
} |
|||
const config = await getConfig(envFile) |
|||
const dbList = await getAllDbs(config["COUCH_DB_URL"]) |
|||
const { Remote, Local } = getPouches(config) |
|||
if (fs.existsSync(TEMP_DIR)) { |
|||
fs.rmSync(TEMP_DIR, { recursive: true }) |
|||
} |
|||
const couchDir = join(TEMP_DIR, COUCH_DIR) |
|||
fs.mkdirSync(TEMP_DIR) |
|||
fs.mkdirSync(couchDir) |
|||
console.log("CouchDB Export") |
|||
const bar = progressBar(dbList.length) |
|||
let count = 0 |
|||
for (let db of dbList) { |
|||
bar.update(++count) |
|||
const remote = new Remote(db) |
|||
const local = new Local(join(TEMP_DIR, COUCH_DIR, db)) |
|||
await replication(remote, local) |
|||
} |
|||
bar.stop() |
|||
console.log("S3 Export") |
|||
await exportObjects() |
|||
tar.create( |
|||
{ |
|||
sync: true, |
|||
gzip: true, |
|||
file: filename, |
|||
cwd: join(TEMP_DIR), |
|||
}, |
|||
[COUCH_DIR, MINIO_DIR] |
|||
) |
|||
fs.rmSync(TEMP_DIR, { recursive: true }) |
|||
console.log(`Generated export file - ${filename}`) |
|||
} |
|||
|
|||
async function importBackup(opts) { |
|||
const envFile = opts.env || undefined |
|||
const filename = opts["import"] || opts |
|||
const config = await getConfig(envFile) |
|||
if (!filename || !fs.existsSync(filename)) { |
|||
console.error("Cannot import without specifying a valid file to import") |
|||
process.exit(-1) |
|||
} |
|||
if (fs.existsSync(TEMP_DIR)) { |
|||
fs.rmSync(TEMP_DIR, { recursive: true }) |
|||
} |
|||
fs.mkdirSync(TEMP_DIR) |
|||
tar.extract({ |
|||
sync: true, |
|||
cwd: join(TEMP_DIR), |
|||
file: filename, |
|||
}) |
|||
const { Remote, Local } = getPouches(config) |
|||
const dbList = fs.readdirSync(join(TEMP_DIR, COUCH_DIR)) |
|||
console.log("CouchDB Import") |
|||
const bar = progressBar(dbList.length) |
|||
let count = 0 |
|||
for (let db of dbList) { |
|||
bar.update(++count) |
|||
const remote = new Remote(db) |
|||
const local = new Local(join(TEMP_DIR, COUCH_DIR, db)) |
|||
await replication(local, remote) |
|||
} |
|||
bar.stop() |
|||
console.log("MinIO Import") |
|||
await importObjects() |
|||
console.log("Import complete") |
|||
fs.rmSync(TEMP_DIR, { recursive: true }) |
|||
} |
|||
|
|||
async function pickOne(opts) { |
|||
if (opts["import"]) { |
|||
return importBackup(opts) |
|||
} else if (opts["export"]) { |
|||
return exportBackup(opts) |
|||
} |
|||
} |
|||
|
|||
const command = new Command(`${CommandWords.BACKUPS}`) |
|||
.addHelp( |
|||
"Allows building backups of Budibase, as well as importing a backup to a new instance." |
|||
) |
|||
.addSubOption( |
|||
"--export [filename]", |
|||
"Export a backup from an existing Budibase installation.", |
|||
exportBackup |
|||
) |
|||
.addSubOption( |
|||
"--import [filename]", |
|||
"Import a backup to a new Budibase installation.", |
|||
importBackup |
|||
) |
|||
.addSubOption( |
|||
"--env [envFile]", |
|||
"Provide an environment variable file to configure the CLI.", |
|||
pickOne |
|||
) |
|||
|
|||
exports.command = command |
|||
@ -0,0 +1,63 @@ |
|||
const { |
|||
ObjectStoreBuckets, |
|||
ObjectStore, |
|||
retrieve, |
|||
uploadDirectory, |
|||
makeSureBucketExists, |
|||
} = require("@budibase/backend-core/objectStore") |
|||
const fs = require("fs") |
|||
const { join } = require("path") |
|||
const { TEMP_DIR, MINIO_DIR } = require("./utils") |
|||
const { progressBar } = require("../utils") |
|||
|
|||
const bucketList = Object.values(ObjectStoreBuckets) |
|||
|
|||
exports.exportObjects = async () => { |
|||
const path = join(TEMP_DIR, MINIO_DIR) |
|||
fs.mkdirSync(path) |
|||
let fullList = [] |
|||
for (let bucket of bucketList) { |
|||
const client = ObjectStore(bucket) |
|||
try { |
|||
await client.headBucket().promise() |
|||
} catch (err) { |
|||
continue |
|||
} |
|||
const list = await client.listObjectsV2().promise() |
|||
fullList = fullList.concat(list.Contents.map(el => ({ ...el, bucket }))) |
|||
} |
|||
const bar = progressBar(fullList.length) |
|||
let count = 0 |
|||
for (let object of fullList) { |
|||
const filename = object.Key |
|||
const data = await retrieve(object.bucket, filename) |
|||
const possiblePath = filename.split("/") |
|||
if (possiblePath.length > 1) { |
|||
const dirs = possiblePath.slice(0, possiblePath.length - 1) |
|||
fs.mkdirSync(join(path, object.bucket, ...dirs), { recursive: true }) |
|||
} |
|||
fs.writeFileSync(join(path, object.bucket, ...possiblePath), data) |
|||
bar.update(++count) |
|||
} |
|||
bar.stop() |
|||
} |
|||
|
|||
exports.importObjects = async () => { |
|||
const path = join(TEMP_DIR, MINIO_DIR) |
|||
const buckets = fs.readdirSync(path) |
|||
let total = 0 |
|||
buckets.forEach(bucket => { |
|||
const files = fs.readdirSync(join(path, bucket)) |
|||
total += files.length |
|||
}) |
|||
const bar = progressBar(total) |
|||
let count = 0 |
|||
for (let bucket of buckets) { |
|||
const client = ObjectStore(bucket) |
|||
await makeSureBucketExists(client, bucket) |
|||
const files = await uploadDirectory(bucket, join(path, bucket), "/") |
|||
count += files.length |
|||
bar.update(count) |
|||
} |
|||
bar.stop() |
|||
} |
|||
@ -0,0 +1,88 @@ |
|||
const dotenv = require("dotenv") |
|||
const fs = require("fs") |
|||
const { string } = require("../questions") |
|||
const { getPouch } = require("../core/db") |
|||
|
|||
exports.DEFAULT_COUCH = "http://budibase:budibase@localhost:10000/db/" |
|||
exports.DEFAULT_MINIO = "http://localhost:10000/" |
|||
exports.TEMP_DIR = ".temp" |
|||
exports.COUCH_DIR = "couchdb" |
|||
exports.MINIO_DIR = "minio" |
|||
|
|||
const REQUIRED = [ |
|||
{ value: "MAIN_PORT", default: "10000" }, |
|||
{ value: "COUCH_DB_URL", default: exports.DEFAULT_COUCH }, |
|||
{ value: "MINIO_URL", default: exports.DEFAULT_MINIO }, |
|||
{ value: "MINIO_ACCESS_KEY" }, |
|||
{ value: "MINIO_SECRET_KEY" }, |
|||
] |
|||
|
|||
exports.checkURLs = config => { |
|||
const mainPort = config["MAIN_PORT"], |
|||
username = config["COUCH_DB_USER"], |
|||
password = config["COUCH_DB_PASSWORD"] |
|||
if (!config["COUCH_DB_URL"] && mainPort && username && password) { |
|||
config[ |
|||
"COUCH_DB_URL" |
|||
] = `http://${username}:${password}@localhost:${mainPort}/db/` |
|||
} |
|||
if (!config["MINIO_URL"]) { |
|||
config["MINIO_URL"] = exports.DEFAULT_MINIO |
|||
} |
|||
return config |
|||
} |
|||
|
|||
exports.askQuestions = async () => { |
|||
console.log( |
|||
"*** NOTE: use a .env file to load these parameters repeatedly ***" |
|||
) |
|||
let config = {} |
|||
for (let property of REQUIRED) { |
|||
config[property.value] = await string(property.value, property.default) |
|||
} |
|||
return config |
|||
} |
|||
|
|||
exports.loadEnvironment = path => { |
|||
if (!fs.existsSync(path)) { |
|||
throw "Unable to file specified .env file" |
|||
} |
|||
const env = fs.readFileSync(path, "utf8") |
|||
const config = exports.checkURLs(dotenv.parse(env)) |
|||
for (let required of REQUIRED) { |
|||
if (!config[required.value]) { |
|||
throw `Cannot find "${required.value}" property in .env file` |
|||
} |
|||
} |
|||
return config |
|||
} |
|||
|
|||
// true is the default value passed by commander
|
|||
exports.getConfig = async (envFile = true) => { |
|||
let config |
|||
if (envFile !== true) { |
|||
config = exports.loadEnvironment(envFile) |
|||
} else { |
|||
config = await exports.askQuestions() |
|||
} |
|||
return config |
|||
} |
|||
|
|||
exports.replication = (from, to) => { |
|||
return new Promise((resolve, reject) => { |
|||
from.replicate |
|||
.to(to) |
|||
.on("complete", () => { |
|||
resolve() |
|||
}) |
|||
.on("error", err => { |
|||
reject(err) |
|||
}) |
|||
}) |
|||
} |
|||
|
|||
exports.getPouches = config => { |
|||
const Remote = getPouch(config["COUCH_DB_URL"]) |
|||
const Local = getPouch() |
|||
return { Remote, Local } |
|||
} |
|||
@ -0,0 +1,38 @@ |
|||
const PouchDB = require("pouchdb") |
|||
const { checkSlashesInUrl } = require("../utils") |
|||
const fetch = require("node-fetch") |
|||
|
|||
/** |
|||
* Fully qualified URL including username and password, or nothing for local |
|||
*/ |
|||
exports.getPouch = (url = undefined) => { |
|||
let POUCH_DB_DEFAULTS = {} |
|||
if (!url) { |
|||
POUCH_DB_DEFAULTS = { |
|||
prefix: undefined, |
|||
adapter: "leveldb", |
|||
} |
|||
} else { |
|||
POUCH_DB_DEFAULTS = { |
|||
prefix: url, |
|||
} |
|||
} |
|||
const replicationStream = require("pouchdb-replication-stream") |
|||
PouchDB.plugin(replicationStream.plugin) |
|||
PouchDB.adapter("writableStream", replicationStream.adapters.writableStream) |
|||
return PouchDB.defaults(POUCH_DB_DEFAULTS) |
|||
} |
|||
|
|||
exports.getAllDbs = async url => { |
|||
const response = await fetch( |
|||
checkSlashesInUrl(encodeURI(`${url}/_all_dbs`)), |
|||
{ |
|||
method: "GET", |
|||
} |
|||
) |
|||
if (response.status === 200) { |
|||
return await response.json() |
|||
} else { |
|||
throw "Cannot connect to CouchDB instance" |
|||
} |
|||
} |
|||
@ -1,6 +1,7 @@ |
|||
const analytics = require("./analytics") |
|||
const hosting = require("./hosting") |
|||
const backups = require("./backups") |
|||
|
|||
exports.getCommands = () => { |
|||
return [hosting.command, analytics.command] |
|||
return [hosting.command, analytics.command, backups.command] |
|||
} |
|||
|
|||
@ -0,0 +1,34 @@ |
|||
const os = require("os") |
|||
const { join } = require("path") |
|||
const fs = require("fs") |
|||
const PREBUILDS = "prebuilds" |
|||
const ARCH = `${os.platform()}-${os.arch()}` |
|||
const PREBUILD_DIR = join(process.execPath, "..", PREBUILDS, ARCH) |
|||
|
|||
checkForBinaries() |
|||
|
|||
function checkForBinaries() { |
|||
const readDir = join(__filename, "..", "..", PREBUILDS, ARCH) |
|||
if (fs.existsSync(PREBUILD_DIR) || !fs.existsSync(readDir)) { |
|||
return |
|||
} |
|||
const natives = fs.readdirSync(readDir) |
|||
if (fs.existsSync(readDir)) { |
|||
fs.mkdirSync(PREBUILD_DIR, { recursive: true }) |
|||
for (let native of natives) { |
|||
const filename = `${native.split(".fake")[0]}.node` |
|||
fs.cpSync(join(readDir, native), join(PREBUILD_DIR, filename)) |
|||
} |
|||
} |
|||
} |
|||
|
|||
function cleanup() { |
|||
if (fs.existsSync(PREBUILD_DIR)) { |
|||
fs.rmSync(PREBUILD_DIR, { recursive: true }) |
|||
} |
|||
} |
|||
|
|||
const events = ["exit", "SIGINT", "SIGUSR1", "SIGUSR2", "uncaughtException"] |
|||
events.forEach(event => { |
|||
process.on(event, cleanup) |
|||
}) |
|||
File diff suppressed because it is too large
File diff suppressed because it is too large
File diff suppressed because it is too large
Loading…
Reference in new issue