mirror of https://github.com/Budibase/budibase.git
mike12345567
5 years ago
16 changed files with 303 additions and 365 deletions
@ -1,28 +1,10 @@ |
|||
const { performDump } = require("../../utilities/templates") |
|||
const path = require("path") |
|||
const os = require("os") |
|||
const fs = require("fs-extra") |
|||
const { performBackup } = require("../../utilities/fileSystem") |
|||
|
|||
exports.exportAppDump = async function(ctx) { |
|||
const { appId } = ctx.query |
|||
|
|||
const appname = decodeURI(ctx.query.appname) |
|||
|
|||
const backupsDir = path.join(os.homedir(), ".budibase", "backups") |
|||
fs.ensureDirSync(backupsDir) |
|||
|
|||
const backupIdentifier = `${appname}Backup${new Date().getTime()}.txt` |
|||
|
|||
await performDump({ |
|||
dir: backupsDir, |
|||
appId, |
|||
name: backupIdentifier, |
|||
}) |
|||
|
|||
ctx.status = 200 |
|||
|
|||
const backupFile = path.join(backupsDir, backupIdentifier) |
|||
|
|||
ctx.attachment(backupIdentifier) |
|||
ctx.body = fs.createReadStream(backupFile) |
|||
ctx.body = await performBackup(appId, backupIdentifier) |
|||
} |
|||
|
|||
@ -1,35 +0,0 @@ |
|||
const { ensureDir, constants, copyFile } = require("fs-extra") |
|||
const { join } = require("../centralPath") |
|||
const { budibaseAppsDir } = require("../budibaseDir") |
|||
|
|||
/** |
|||
* Compile all the non-db static web assets that are required for the running of |
|||
* a budibase application. This includes the JSON structure of the DOM and |
|||
* the client library, a script responsible for reading the JSON structure |
|||
* and rendering the application. |
|||
* @param {string} appId id of the application we want to compile static assets for |
|||
*/ |
|||
module.exports = async appId => { |
|||
const publicPath = join(budibaseAppsDir(), appId, "public") |
|||
await ensureDir(publicPath) |
|||
await copyClientLib(publicPath) |
|||
} |
|||
|
|||
/** |
|||
* Copy the budibase client library and sourcemap from NPM to <appId>/public/. |
|||
* The client library is then served as a static asset when the budibase application |
|||
* is running in preview or prod |
|||
* @param {String} publicPath - path to write the client library to |
|||
*/ |
|||
const copyClientLib = async publicPath => { |
|||
const sourcepath = require.resolve("@budibase/client") |
|||
const destPath = join(publicPath, "budibase-client.js") |
|||
|
|||
await copyFile(sourcepath, destPath, constants.COPYFILE_FICLONE) |
|||
|
|||
await copyFile( |
|||
sourcepath + ".map", |
|||
destPath + ".map", |
|||
constants.COPYFILE_FICLONE |
|||
) |
|||
} |
|||
@ -1,29 +0,0 @@ |
|||
const stream = require("stream") |
|||
const fetch = require("node-fetch") |
|||
const tar = require("tar-fs") |
|||
const zlib = require("zlib") |
|||
const { promisify } = require("util") |
|||
const packageJson = require("../../package.json") |
|||
|
|||
const streamPipeline = promisify(stream.pipeline) |
|||
|
|||
// can't really test this due to the downloading nature of it, wouldn't be a great test case
|
|||
/* istanbul ignore next */ |
|||
exports.downloadExtractComponentLibraries = async appFolder => { |
|||
const LIBRARIES = ["standard-components"] |
|||
|
|||
// Need to download tarballs directly from NPM as our users may not have node on their machine
|
|||
for (let lib of LIBRARIES) { |
|||
// download tarball
|
|||
const registryUrl = `https://registry.npmjs.org/@budibase/${lib}/-/${lib}-${packageJson.version}.tgz` |
|||
const response = await fetch(registryUrl) |
|||
if (!response.ok) |
|||
throw new Error(`unexpected response ${response.statusText}`) |
|||
|
|||
await streamPipeline( |
|||
response.body, |
|||
zlib.Unzip(), |
|||
tar.extract(`${appFolder}/node_modules/@budibase/${lib}`) |
|||
) |
|||
} |
|||
} |
|||
@ -1,40 +0,0 @@ |
|||
const { budibaseTempDir } = require("./budibaseDir") |
|||
const { isDev } = require("./index") |
|||
const fs = require("fs") |
|||
const { join } = require("path") |
|||
const { downloadTemplate } = require("./templates") |
|||
|
|||
/** |
|||
* The single stack system (Cloud and Builder) should not make use of the file system where possible, |
|||
* this file handles all of the file access for the system with the intention of limiting it all to one |
|||
* place. Keeping all of this logic in one place means that when we need to do file system access (like |
|||
* downloading a package or opening a temporary file) in can be done in way that we can confirm it shouldn't |
|||
* be done through an object store instead. |
|||
*/ |
|||
|
|||
/** |
|||
* Checks if the system is currently in development mode and if it is makes sure |
|||
* everything required to function is ready. |
|||
*/ |
|||
exports.checkDevelopmentEnvironment = () => { |
|||
if (isDev() && !fs.existsSync(budibaseTempDir())) { |
|||
console.error( |
|||
"Please run a build before attempting to run server independently to fill 'tmp' directory." |
|||
) |
|||
process.exit(-1) |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* This function manages temporary template files which are stored by Koa. |
|||
* @param {Object} template The template object retrieved from the Koa context object. |
|||
* @returns {Object} Returns an fs read stream which can be loaded into the database. |
|||
*/ |
|||
exports.getTemplateStream = async template => { |
|||
if (template.file) { |
|||
return fs.createReadStream(template.file.path) |
|||
} else { |
|||
const templatePath = await downloadTemplate(...template.key.split("/")) |
|||
return fs.createReadStream(join(templatePath, "db", "dump.txt")) |
|||
} |
|||
} |
|||
@ -0,0 +1,98 @@ |
|||
const { budibaseTempDir } = require("../budibaseDir") |
|||
const { isDev } = require("../index") |
|||
const fs = require("fs") |
|||
const { join } = require("path") |
|||
const uuid = require("uuid/v4") |
|||
const CouchDB = require("../../db") |
|||
const { ObjectStoreBuckets } = require("../../constants") |
|||
const { streamUpload, deleteFolder, downloadTarball } = require("./utilities") |
|||
const { downloadLibraries, newAppPublicPath } = require("./newApp") |
|||
|
|||
/** |
|||
* The single stack system (Cloud and Builder) should not make use of the file system where possible, |
|||
* this file handles all of the file access for the system with the intention of limiting it all to one |
|||
* place. Keeping all of this logic in one place means that when we need to do file system access (like |
|||
* downloading a package or opening a temporary file) in can be done in way that we can confirm it shouldn't |
|||
* be done through an object store instead. |
|||
*/ |
|||
|
|||
/** |
|||
* Checks if the system is currently in development mode and if it is makes sure |
|||
* everything required to function is ready. |
|||
*/ |
|||
exports.checkDevelopmentEnvironment = () => { |
|||
if (isDev() && !fs.existsSync(budibaseTempDir())) { |
|||
console.error( |
|||
"Please run a build before attempting to run server independently to fill 'tmp' directory." |
|||
) |
|||
process.exit(-1) |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* This function manages temporary template files which are stored by Koa. |
|||
* @param {Object} template The template object retrieved from the Koa context object. |
|||
* @returns {Object} Returns an fs read stream which can be loaded into the database. |
|||
*/ |
|||
exports.getTemplateStream = async template => { |
|||
if (template.file) { |
|||
return fs.createReadStream(template.file.path) |
|||
} else { |
|||
const tmpPath = await exports.downloadTemplate(...template.key.split("/")) |
|||
return fs.createReadStream(join(tmpPath, "db", "dump.txt")) |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* Used to retrieve a handlebars file from the system which will be used as a template. |
|||
* This is allowable as the template handlebars files should be static and identical across |
|||
* the cluster. |
|||
* @param {string} path The path to the handlebars file which is to be loaded. |
|||
* @returns {string} The loaded handlebars file as a string - loaded as utf8. |
|||
*/ |
|||
exports.loadHandlebarsFile = path => { |
|||
return fs.readFileSync(path, "utf8") |
|||
} |
|||
|
|||
/** |
|||
* When return a file from the API need to write the file to the system temporarily so we |
|||
* can create a read stream to send. |
|||
* @param {string} contents the contents of the file which is to be returned from the API. |
|||
* @return {Object} the read stream which can be put into the koa context body. |
|||
*/ |
|||
exports.apiFileReturn = contents => { |
|||
const path = join(budibaseTempDir(), uuid()) |
|||
fs.writeFileSync(path, contents) |
|||
return fs.createReadStream(path) |
|||
} |
|||
|
|||
exports.performBackup = async (appId, backupName) => { |
|||
const path = join(budibaseTempDir(), backupName) |
|||
const writeStream = fs.createWriteStream(path) |
|||
// perform couch dump
|
|||
const instanceDb = new CouchDB(appId) |
|||
await instanceDb.dump(writeStream, {}) |
|||
// write the file to the object store
|
|||
await streamUpload( |
|||
ObjectStoreBuckets.BACKUPS, |
|||
join(appId, backupName), |
|||
fs.createReadStream(path) |
|||
) |
|||
return fs.createReadStream(path) |
|||
} |
|||
|
|||
exports.createApp = async appId => { |
|||
await downloadLibraries(appId) |
|||
await newAppPublicPath(appId) |
|||
} |
|||
|
|||
exports.deleteApp = async appId => { |
|||
await deleteFolder(ObjectStoreBuckets.APPS, `${appId}/`) |
|||
} |
|||
|
|||
exports.downloadTemplate = async (type, name) => { |
|||
const DEFAULT_TEMPLATES_BUCKET = |
|||
"prod-budi-templates.s3-eu-west-1.amazonaws.com" |
|||
const templateUrl = `https://${DEFAULT_TEMPLATES_BUCKET}/templates/${type}/${name}.tar.gz` |
|||
return downloadTarball(templateUrl, ObjectStoreBuckets.TEMPLATES, type) |
|||
} |
|||
@ -0,0 +1,34 @@ |
|||
const packageJson = require("../../../package.json") |
|||
const { join } = require("path") |
|||
const { ObjectStoreBuckets } = require("../../constants") |
|||
const { streamUpload, downloadTarball } = require("./utilities") |
|||
const fs = require("fs") |
|||
|
|||
const BUCKET_NAME = ObjectStoreBuckets.APPS |
|||
|
|||
// can't really test this due to the downloading nature of it, wouldn't be a great test case
|
|||
/* istanbul ignore next */ |
|||
exports.downloadLibraries = async appId => { |
|||
const LIBRARIES = ["standard-components"] |
|||
|
|||
// Need to download tarballs directly from NPM as our users may not have node on their machine
|
|||
for (let lib of LIBRARIES) { |
|||
// download tarball
|
|||
const registryUrl = `https://registry.npmjs.org/@budibase/${lib}/-/${lib}-${packageJson.version}.tgz` |
|||
const path = join(appId, "node_modules", "@budibase", lib) |
|||
await downloadTarball(registryUrl, BUCKET_NAME, path) |
|||
} |
|||
} |
|||
|
|||
exports.newAppPublicPath = async appId => { |
|||
const path = join(appId, "public") |
|||
const sourcepath = require.resolve("@budibase/client") |
|||
const destPath = join(path, "budibase-client.js") |
|||
|
|||
await streamUpload(BUCKET_NAME, destPath, fs.createReadStream(sourcepath)) |
|||
await streamUpload( |
|||
BUCKET_NAME, |
|||
destPath + ".map", |
|||
fs.createReadStream(sourcepath + ".map") |
|||
) |
|||
} |
|||
@ -0,0 +1,116 @@ |
|||
const sanitize = require("sanitize-s3-objectkey") |
|||
const AWS = require("aws-sdk") |
|||
const stream = require("stream") |
|||
const fetch = require("node-fetch") |
|||
const tar = require("tar-fs") |
|||
const zlib = require("zlib") |
|||
const { promisify } = require("util") |
|||
const { join } = require("path") |
|||
const { streamUpload } = require("./utilities") |
|||
const fs = require("fs") |
|||
const { budibaseTempDir } = require("../budibaseDir") |
|||
|
|||
const streamPipeline = promisify(stream.pipeline) |
|||
|
|||
/** |
|||
* Gets a connection to the object store using the S3 SDK. |
|||
* @param {string} bucket the name of the bucket which blobs will be uploaded/retrieved from. |
|||
* @return {Object} an S3 object store object, check S3 Nodejs SDK for usage. |
|||
* @constructor |
|||
*/ |
|||
exports.ObjectStore = bucket => { |
|||
return new AWS.S3({ |
|||
params: { |
|||
Bucket: bucket, |
|||
}, |
|||
}) |
|||
} |
|||
|
|||
exports.makeSureBucketExists = async (client, bucketName) => { |
|||
try { |
|||
await client |
|||
.headBucket({ |
|||
Bucket: bucketName, |
|||
}) |
|||
.promise() |
|||
} catch (err) { |
|||
// bucket doesn't exist create it
|
|||
if (err.statusCode === 404) { |
|||
await client |
|||
.createBucket({ |
|||
Bucket: bucketName, |
|||
}) |
|||
.promise() |
|||
} else { |
|||
throw err |
|||
} |
|||
} |
|||
} |
|||
|
|||
exports.streamUpload = async (bucket, filename, stream) => { |
|||
const objectStore = exports.ObjectStore(bucket) |
|||
await exports.makeSureBucketExists(objectStore, bucket) |
|||
|
|||
const params = { |
|||
Bucket: bucket, |
|||
Key: sanitize(filename).replace(/\\/g, "/"), |
|||
Body: stream, |
|||
} |
|||
return objectStore.upload(params).promise() |
|||
} |
|||
|
|||
exports.deleteFolder = async (bucket, folder) => { |
|||
const client = exports.ObjectStore(bucket) |
|||
const listParams = { |
|||
Bucket: bucket, |
|||
Prefix: folder, |
|||
} |
|||
|
|||
const data = await client.listObjects(listParams).promise() |
|||
if (data.Contents.length > 0) { |
|||
const deleteParams = { |
|||
Bucket: bucket, |
|||
Delete: { |
|||
Objects: [], |
|||
}, |
|||
} |
|||
|
|||
data.Contents.forEach(content => { |
|||
deleteParams.Delete.Objects.push({ Key: content.Key }) |
|||
}) |
|||
|
|||
const data = await client.deleteObjects(deleteParams).promise() |
|||
// can only empty 1000 items at once
|
|||
if (data.Contents.length === 1000) { |
|||
return exports.deleteFolder(bucket, folder) |
|||
} |
|||
} |
|||
} |
|||
|
|||
exports.uploadDirectory = async (bucket, localPath, bucketPath) => { |
|||
let uploads = [] |
|||
const files = fs.readdirSync(localPath, { withFileTypes: true }) |
|||
for (let file of files) { |
|||
const path = join(bucketPath, file.name) |
|||
const local = join(localPath, file.name) |
|||
if (file.isDirectory()) { |
|||
uploads.push(exports.uploadDirectory(bucket, local, path)) |
|||
} else { |
|||
uploads.push(streamUpload(bucket, path, fs.createReadStream(local))) |
|||
} |
|||
} |
|||
await Promise.all(uploads) |
|||
} |
|||
|
|||
exports.downloadTarball = async (url, bucket, path) => { |
|||
const response = await fetch(url) |
|||
if (!response.ok) { |
|||
throw new Error(`unexpected response ${response.statusText}`) |
|||
} |
|||
|
|||
const tmpPath = join(budibaseTempDir(), path) |
|||
await streamPipeline(response.body, zlib.Unzip(), tar.extract(tmpPath)) |
|||
await exports.uploadDirectory(bucket, tmpPath, path) |
|||
// return the temporary path incase there is a use for it
|
|||
return tmpPath |
|||
} |
|||
Loading…
Reference in new issue