forked from tsai/budibase
mike12345567
6 years ago
16 changed files with 303 additions and 365 deletions
@ -1,28 +1,10 @@ |
|||||
const { performDump } = require("../../utilities/templates") |
const { performBackup } = require("../../utilities/fileSystem") |
||||
const path = require("path") |
|
||||
const os = require("os") |
|
||||
const fs = require("fs-extra") |
|
||||
|
|
||||
exports.exportAppDump = async function(ctx) { |
exports.exportAppDump = async function(ctx) { |
||||
const { appId } = ctx.query |
const { appId } = ctx.query |
||||
|
|
||||
const appname = decodeURI(ctx.query.appname) |
const appname = decodeURI(ctx.query.appname) |
||||
|
|
||||
const backupsDir = path.join(os.homedir(), ".budibase", "backups") |
|
||||
fs.ensureDirSync(backupsDir) |
|
||||
|
|
||||
const backupIdentifier = `${appname}Backup${new Date().getTime()}.txt` |
const backupIdentifier = `${appname}Backup${new Date().getTime()}.txt` |
||||
|
|
||||
await performDump({ |
|
||||
dir: backupsDir, |
|
||||
appId, |
|
||||
name: backupIdentifier, |
|
||||
}) |
|
||||
|
|
||||
ctx.status = 200 |
|
||||
|
|
||||
const backupFile = path.join(backupsDir, backupIdentifier) |
|
||||
|
|
||||
ctx.attachment(backupIdentifier) |
ctx.attachment(backupIdentifier) |
||||
ctx.body = fs.createReadStream(backupFile) |
ctx.body = await performBackup(appId, backupIdentifier) |
||||
} |
} |
||||
|
|||||
@ -1,35 +0,0 @@ |
|||||
const { ensureDir, constants, copyFile } = require("fs-extra") |
|
||||
const { join } = require("../centralPath") |
|
||||
const { budibaseAppsDir } = require("../budibaseDir") |
|
||||
|
|
||||
/** |
|
||||
* Compile all the non-db static web assets that are required for the running of |
|
||||
* a budibase application. This includes the JSON structure of the DOM and |
|
||||
* the client library, a script responsible for reading the JSON structure |
|
||||
* and rendering the application. |
|
||||
* @param {string} appId id of the application we want to compile static assets for |
|
||||
*/ |
|
||||
module.exports = async appId => { |
|
||||
const publicPath = join(budibaseAppsDir(), appId, "public") |
|
||||
await ensureDir(publicPath) |
|
||||
await copyClientLib(publicPath) |
|
||||
} |
|
||||
|
|
||||
/** |
|
||||
* Copy the budibase client library and sourcemap from NPM to <appId>/public/. |
|
||||
* The client library is then served as a static asset when the budibase application |
|
||||
* is running in preview or prod |
|
||||
* @param {String} publicPath - path to write the client library to |
|
||||
*/ |
|
||||
const copyClientLib = async publicPath => { |
|
||||
const sourcepath = require.resolve("@budibase/client") |
|
||||
const destPath = join(publicPath, "budibase-client.js") |
|
||||
|
|
||||
await copyFile(sourcepath, destPath, constants.COPYFILE_FICLONE) |
|
||||
|
|
||||
await copyFile( |
|
||||
sourcepath + ".map", |
|
||||
destPath + ".map", |
|
||||
constants.COPYFILE_FICLONE |
|
||||
) |
|
||||
} |
|
||||
@ -1,29 +0,0 @@ |
|||||
const stream = require("stream") |
|
||||
const fetch = require("node-fetch") |
|
||||
const tar = require("tar-fs") |
|
||||
const zlib = require("zlib") |
|
||||
const { promisify } = require("util") |
|
||||
const packageJson = require("../../package.json") |
|
||||
|
|
||||
const streamPipeline = promisify(stream.pipeline) |
|
||||
|
|
||||
// can't really test this due to the downloading nature of it, wouldn't be a great test case
|
|
||||
/* istanbul ignore next */ |
|
||||
exports.downloadExtractComponentLibraries = async appFolder => { |
|
||||
const LIBRARIES = ["standard-components"] |
|
||||
|
|
||||
// Need to download tarballs directly from NPM as our users may not have node on their machine
|
|
||||
for (let lib of LIBRARIES) { |
|
||||
// download tarball
|
|
||||
const registryUrl = `https://registry.npmjs.org/@budibase/${lib}/-/${lib}-${packageJson.version}.tgz` |
|
||||
const response = await fetch(registryUrl) |
|
||||
if (!response.ok) |
|
||||
throw new Error(`unexpected response ${response.statusText}`) |
|
||||
|
|
||||
await streamPipeline( |
|
||||
response.body, |
|
||||
zlib.Unzip(), |
|
||||
tar.extract(`${appFolder}/node_modules/@budibase/${lib}`) |
|
||||
) |
|
||||
} |
|
||||
} |
|
||||
@ -1,40 +0,0 @@ |
|||||
const { budibaseTempDir } = require("./budibaseDir") |
|
||||
const { isDev } = require("./index") |
|
||||
const fs = require("fs") |
|
||||
const { join } = require("path") |
|
||||
const { downloadTemplate } = require("./templates") |
|
||||
|
|
||||
/** |
|
||||
* The single stack system (Cloud and Builder) should not make use of the file system where possible, |
|
||||
* this file handles all of the file access for the system with the intention of limiting it all to one |
|
||||
* place. Keeping all of this logic in one place means that when we need to do file system access (like |
|
||||
* downloading a package or opening a temporary file) in can be done in way that we can confirm it shouldn't |
|
||||
* be done through an object store instead. |
|
||||
*/ |
|
||||
|
|
||||
/** |
|
||||
* Checks if the system is currently in development mode and if it is makes sure |
|
||||
* everything required to function is ready. |
|
||||
*/ |
|
||||
exports.checkDevelopmentEnvironment = () => { |
|
||||
if (isDev() && !fs.existsSync(budibaseTempDir())) { |
|
||||
console.error( |
|
||||
"Please run a build before attempting to run server independently to fill 'tmp' directory." |
|
||||
) |
|
||||
process.exit(-1) |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
/** |
|
||||
* This function manages temporary template files which are stored by Koa. |
|
||||
* @param {Object} template The template object retrieved from the Koa context object. |
|
||||
* @returns {Object} Returns an fs read stream which can be loaded into the database. |
|
||||
*/ |
|
||||
exports.getTemplateStream = async template => { |
|
||||
if (template.file) { |
|
||||
return fs.createReadStream(template.file.path) |
|
||||
} else { |
|
||||
const templatePath = await downloadTemplate(...template.key.split("/")) |
|
||||
return fs.createReadStream(join(templatePath, "db", "dump.txt")) |
|
||||
} |
|
||||
} |
|
||||
@ -0,0 +1,98 @@ |
|||||
|
const { budibaseTempDir } = require("../budibaseDir") |
||||
|
const { isDev } = require("../index") |
||||
|
const fs = require("fs") |
||||
|
const { join } = require("path") |
||||
|
const uuid = require("uuid/v4") |
||||
|
const CouchDB = require("../../db") |
||||
|
const { ObjectStoreBuckets } = require("../../constants") |
||||
|
const { streamUpload, deleteFolder, downloadTarball } = require("./utilities") |
||||
|
const { downloadLibraries, newAppPublicPath } = require("./newApp") |
||||
|
|
||||
|
/** |
||||
|
* The single stack system (Cloud and Builder) should not make use of the file system where possible, |
||||
|
* this file handles all of the file access for the system with the intention of limiting it all to one |
||||
|
* place. Keeping all of this logic in one place means that when we need to do file system access (like |
||||
|
* downloading a package or opening a temporary file) in can be done in way that we can confirm it shouldn't |
||||
|
* be done through an object store instead. |
||||
|
*/ |
||||
|
|
||||
|
/** |
||||
|
* Checks if the system is currently in development mode and if it is makes sure |
||||
|
* everything required to function is ready. |
||||
|
*/ |
||||
|
exports.checkDevelopmentEnvironment = () => { |
||||
|
if (isDev() && !fs.existsSync(budibaseTempDir())) { |
||||
|
console.error( |
||||
|
"Please run a build before attempting to run server independently to fill 'tmp' directory." |
||||
|
) |
||||
|
process.exit(-1) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* This function manages temporary template files which are stored by Koa. |
||||
|
* @param {Object} template The template object retrieved from the Koa context object. |
||||
|
* @returns {Object} Returns an fs read stream which can be loaded into the database. |
||||
|
*/ |
||||
|
exports.getTemplateStream = async template => { |
||||
|
if (template.file) { |
||||
|
return fs.createReadStream(template.file.path) |
||||
|
} else { |
||||
|
const tmpPath = await exports.downloadTemplate(...template.key.split("/")) |
||||
|
return fs.createReadStream(join(tmpPath, "db", "dump.txt")) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* Used to retrieve a handlebars file from the system which will be used as a template. |
||||
|
* This is allowable as the template handlebars files should be static and identical across |
||||
|
* the cluster. |
||||
|
* @param {string} path The path to the handlebars file which is to be loaded. |
||||
|
* @returns {string} The loaded handlebars file as a string - loaded as utf8. |
||||
|
*/ |
||||
|
exports.loadHandlebarsFile = path => { |
||||
|
return fs.readFileSync(path, "utf8") |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* When return a file from the API need to write the file to the system temporarily so we |
||||
|
* can create a read stream to send. |
||||
|
* @param {string} contents the contents of the file which is to be returned from the API. |
||||
|
* @return {Object} the read stream which can be put into the koa context body. |
||||
|
*/ |
||||
|
exports.apiFileReturn = contents => { |
||||
|
const path = join(budibaseTempDir(), uuid()) |
||||
|
fs.writeFileSync(path, contents) |
||||
|
return fs.createReadStream(path) |
||||
|
} |
||||
|
|
||||
|
exports.performBackup = async (appId, backupName) => { |
||||
|
const path = join(budibaseTempDir(), backupName) |
||||
|
const writeStream = fs.createWriteStream(path) |
||||
|
// perform couch dump
|
||||
|
const instanceDb = new CouchDB(appId) |
||||
|
await instanceDb.dump(writeStream, {}) |
||||
|
// write the file to the object store
|
||||
|
await streamUpload( |
||||
|
ObjectStoreBuckets.BACKUPS, |
||||
|
join(appId, backupName), |
||||
|
fs.createReadStream(path) |
||||
|
) |
||||
|
return fs.createReadStream(path) |
||||
|
} |
||||
|
|
||||
|
exports.createApp = async appId => { |
||||
|
await downloadLibraries(appId) |
||||
|
await newAppPublicPath(appId) |
||||
|
} |
||||
|
|
||||
|
exports.deleteApp = async appId => { |
||||
|
await deleteFolder(ObjectStoreBuckets.APPS, `${appId}/`) |
||||
|
} |
||||
|
|
||||
|
exports.downloadTemplate = async (type, name) => { |
||||
|
const DEFAULT_TEMPLATES_BUCKET = |
||||
|
"prod-budi-templates.s3-eu-west-1.amazonaws.com" |
||||
|
const templateUrl = `https://${DEFAULT_TEMPLATES_BUCKET}/templates/${type}/${name}.tar.gz` |
||||
|
return downloadTarball(templateUrl, ObjectStoreBuckets.TEMPLATES, type) |
||||
|
} |
||||
@ -0,0 +1,34 @@ |
|||||
|
const packageJson = require("../../../package.json") |
||||
|
const { join } = require("path") |
||||
|
const { ObjectStoreBuckets } = require("../../constants") |
||||
|
const { streamUpload, downloadTarball } = require("./utilities") |
||||
|
const fs = require("fs") |
||||
|
|
||||
|
const BUCKET_NAME = ObjectStoreBuckets.APPS |
||||
|
|
||||
|
// can't really test this due to the downloading nature of it, wouldn't be a great test case
|
||||
|
/* istanbul ignore next */ |
||||
|
exports.downloadLibraries = async appId => { |
||||
|
const LIBRARIES = ["standard-components"] |
||||
|
|
||||
|
// Need to download tarballs directly from NPM as our users may not have node on their machine
|
||||
|
for (let lib of LIBRARIES) { |
||||
|
// download tarball
|
||||
|
const registryUrl = `https://registry.npmjs.org/@budibase/${lib}/-/${lib}-${packageJson.version}.tgz` |
||||
|
const path = join(appId, "node_modules", "@budibase", lib) |
||||
|
await downloadTarball(registryUrl, BUCKET_NAME, path) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
exports.newAppPublicPath = async appId => { |
||||
|
const path = join(appId, "public") |
||||
|
const sourcepath = require.resolve("@budibase/client") |
||||
|
const destPath = join(path, "budibase-client.js") |
||||
|
|
||||
|
await streamUpload(BUCKET_NAME, destPath, fs.createReadStream(sourcepath)) |
||||
|
await streamUpload( |
||||
|
BUCKET_NAME, |
||||
|
destPath + ".map", |
||||
|
fs.createReadStream(sourcepath + ".map") |
||||
|
) |
||||
|
} |
||||
@ -0,0 +1,116 @@ |
|||||
|
const sanitize = require("sanitize-s3-objectkey") |
||||
|
const AWS = require("aws-sdk") |
||||
|
const stream = require("stream") |
||||
|
const fetch = require("node-fetch") |
||||
|
const tar = require("tar-fs") |
||||
|
const zlib = require("zlib") |
||||
|
const { promisify } = require("util") |
||||
|
const { join } = require("path") |
||||
|
const { streamUpload } = require("./utilities") |
||||
|
const fs = require("fs") |
||||
|
const { budibaseTempDir } = require("../budibaseDir") |
||||
|
|
||||
|
const streamPipeline = promisify(stream.pipeline) |
||||
|
|
||||
|
/** |
||||
|
* Gets a connection to the object store using the S3 SDK. |
||||
|
* @param {string} bucket the name of the bucket which blobs will be uploaded/retrieved from. |
||||
|
* @return {Object} an S3 object store object, check S3 Nodejs SDK for usage. |
||||
|
* @constructor |
||||
|
*/ |
||||
|
exports.ObjectStore = bucket => { |
||||
|
return new AWS.S3({ |
||||
|
params: { |
||||
|
Bucket: bucket, |
||||
|
}, |
||||
|
}) |
||||
|
} |
||||
|
|
||||
|
exports.makeSureBucketExists = async (client, bucketName) => { |
||||
|
try { |
||||
|
await client |
||||
|
.headBucket({ |
||||
|
Bucket: bucketName, |
||||
|
}) |
||||
|
.promise() |
||||
|
} catch (err) { |
||||
|
// bucket doesn't exist create it
|
||||
|
if (err.statusCode === 404) { |
||||
|
await client |
||||
|
.createBucket({ |
||||
|
Bucket: bucketName, |
||||
|
}) |
||||
|
.promise() |
||||
|
} else { |
||||
|
throw err |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
exports.streamUpload = async (bucket, filename, stream) => { |
||||
|
const objectStore = exports.ObjectStore(bucket) |
||||
|
await exports.makeSureBucketExists(objectStore, bucket) |
||||
|
|
||||
|
const params = { |
||||
|
Bucket: bucket, |
||||
|
Key: sanitize(filename).replace(/\\/g, "/"), |
||||
|
Body: stream, |
||||
|
} |
||||
|
return objectStore.upload(params).promise() |
||||
|
} |
||||
|
|
||||
|
exports.deleteFolder = async (bucket, folder) => { |
||||
|
const client = exports.ObjectStore(bucket) |
||||
|
const listParams = { |
||||
|
Bucket: bucket, |
||||
|
Prefix: folder, |
||||
|
} |
||||
|
|
||||
|
const data = await client.listObjects(listParams).promise() |
||||
|
if (data.Contents.length > 0) { |
||||
|
const deleteParams = { |
||||
|
Bucket: bucket, |
||||
|
Delete: { |
||||
|
Objects: [], |
||||
|
}, |
||||
|
} |
||||
|
|
||||
|
data.Contents.forEach(content => { |
||||
|
deleteParams.Delete.Objects.push({ Key: content.Key }) |
||||
|
}) |
||||
|
|
||||
|
const data = await client.deleteObjects(deleteParams).promise() |
||||
|
// can only empty 1000 items at once
|
||||
|
if (data.Contents.length === 1000) { |
||||
|
return exports.deleteFolder(bucket, folder) |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
exports.uploadDirectory = async (bucket, localPath, bucketPath) => { |
||||
|
let uploads = [] |
||||
|
const files = fs.readdirSync(localPath, { withFileTypes: true }) |
||||
|
for (let file of files) { |
||||
|
const path = join(bucketPath, file.name) |
||||
|
const local = join(localPath, file.name) |
||||
|
if (file.isDirectory()) { |
||||
|
uploads.push(exports.uploadDirectory(bucket, local, path)) |
||||
|
} else { |
||||
|
uploads.push(streamUpload(bucket, path, fs.createReadStream(local))) |
||||
|
} |
||||
|
} |
||||
|
await Promise.all(uploads) |
||||
|
} |
||||
|
|
||||
|
exports.downloadTarball = async (url, bucket, path) => { |
||||
|
const response = await fetch(url) |
||||
|
if (!response.ok) { |
||||
|
throw new Error(`unexpected response ${response.statusText}`) |
||||
|
} |
||||
|
|
||||
|
const tmpPath = join(budibaseTempDir(), path) |
||||
|
await streamPipeline(response.body, zlib.Unzip(), tar.extract(tmpPath)) |
||||
|
await exports.uploadDirectory(bucket, tmpPath, path) |
||||
|
// return the temporary path incase there is a use for it
|
||||
|
return tmpPath |
||||
|
} |
||||
Loading…
Reference in new issue