forked from tsai/budibase
Martin McKeaveney
6 years ago
72 changed files with 922 additions and 874 deletions
@ -1,17 +0,0 @@ |
|||||
# url of couch db, including username and password |
|
||||
# http://admin:password@localhost:5984 |
|
||||
COUCH_DB_URL={{couchDbUrl}} |
|
||||
|
|
||||
# identifies a client database - i.e. group of apps |
|
||||
CLIENT_ID={{clientId}} |
|
||||
|
|
||||
# used to create cookie hashes |
|
||||
JWT_SECRET={{cookieKey1}} |
|
||||
|
|
||||
# error level for koa-pino |
|
||||
LOG_LEVEL=info |
|
||||
|
|
||||
DEPLOYMENT_CREDENTIALS_URL="https://dt4mpwwap8.execute-api.eu-west-1.amazonaws.com/prod/" |
|
||||
DEPLOYMENT_DB_URL="https://couchdb.budi.live:5984" |
|
||||
SENTRY_DSN=https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131 |
|
||||
ENABLE_ANALYTICS="true" |
|
||||
@ -1,56 +1,32 @@ |
|||||
const fs = require("fs") |
const builderDB = require("../../db/builder") |
||||
const { join } = require("../../utilities/centralPath") |
|
||||
const readline = require("readline") |
|
||||
const { budibaseAppsDir } = require("../../utilities/budibaseDir") |
|
||||
const env = require("../../environment") |
|
||||
const ENV_FILE_PATH = "/.env" |
|
||||
|
|
||||
exports.fetch = async function(ctx) { |
exports.fetch = async function(ctx) { |
||||
ctx.status = 200 |
try { |
||||
ctx.body = { |
const mainDoc = await builderDB.getBuilderMainDoc() |
||||
budibase: env.BUDIBASE_API_KEY, |
ctx.body = mainDoc.apiKeys ? mainDoc.apiKeys : {} |
||||
userId: env.USERID_API_KEY, |
} catch (err) { |
||||
|
/* istanbul ignore next */ |
||||
|
ctx.throw(400, err) |
||||
} |
} |
||||
} |
} |
||||
|
|
||||
exports.update = async function(ctx) { |
exports.update = async function(ctx) { |
||||
const key = `${ctx.params.key.toUpperCase()}_API_KEY` |
const key = ctx.params.key |
||||
const value = ctx.request.body.value |
const value = ctx.request.body.value |
||||
|
|
||||
// set environment variables
|
try { |
||||
env._set(key, value) |
const mainDoc = await builderDB.getBuilderMainDoc() |
||||
|
if (mainDoc.apiKeys == null) { |
||||
// Write to file
|
mainDoc.apiKeys = {} |
||||
await updateValues([key, value]) |
|
||||
|
|
||||
ctx.status = 200 |
|
||||
ctx.message = `Updated ${ctx.params.key} API key succesfully.` |
|
||||
ctx.body = { [ctx.params.key]: ctx.request.body.value } |
|
||||
} |
|
||||
|
|
||||
async function updateValues([key, value]) { |
|
||||
let newContent = "" |
|
||||
let keyExists = false |
|
||||
let envPath = join(budibaseAppsDir(), ENV_FILE_PATH) |
|
||||
const readInterface = readline.createInterface({ |
|
||||
input: fs.createReadStream(envPath), |
|
||||
output: process.stdout, |
|
||||
console: false, |
|
||||
}) |
|
||||
readInterface.on("line", function(line) { |
|
||||
// Mutate lines and change API Key
|
|
||||
if (line.startsWith(key)) { |
|
||||
line = `${key}=${value}` |
|
||||
keyExists = true |
|
||||
} |
} |
||||
newContent = `${newContent}\n${line}` |
mainDoc.apiKeys[key] = value |
||||
}) |
const resp = await builderDB.setBuilderMainDoc(mainDoc) |
||||
readInterface.on("close", function() { |
ctx.body = { |
||||
// Write file here
|
_id: resp.id, |
||||
if (!keyExists) { |
_rev: resp.rev, |
||||
// Add API Key if it doesn't exist in the file at all
|
|
||||
newContent = `${newContent}\n${key}=${value}` |
|
||||
} |
} |
||||
fs.writeFileSync(envPath, newContent) |
} catch (err) { |
||||
}) |
/* istanbul ignore next */ |
||||
|
ctx.throw(400, err) |
||||
|
} |
||||
} |
} |
||||
|
|||||
@ -1,28 +1,10 @@ |
|||||
const { performDump } = require("../../utilities/templates") |
const { performBackup } = require("../../utilities/fileSystem") |
||||
const path = require("path") |
|
||||
const os = require("os") |
|
||||
const fs = require("fs-extra") |
|
||||
|
|
||||
exports.exportAppDump = async function(ctx) { |
exports.exportAppDump = async function(ctx) { |
||||
const { appId } = ctx.query |
const { appId } = ctx.query |
||||
|
|
||||
const appname = decodeURI(ctx.query.appname) |
const appname = decodeURI(ctx.query.appname) |
||||
|
|
||||
const backupsDir = path.join(os.homedir(), ".budibase", "backups") |
|
||||
fs.ensureDirSync(backupsDir) |
|
||||
|
|
||||
const backupIdentifier = `${appname}Backup${new Date().getTime()}.txt` |
const backupIdentifier = `${appname}Backup${new Date().getTime()}.txt` |
||||
|
|
||||
await performDump({ |
|
||||
dir: backupsDir, |
|
||||
appId, |
|
||||
name: backupIdentifier, |
|
||||
}) |
|
||||
|
|
||||
ctx.status = 200 |
|
||||
|
|
||||
const backupFile = path.join(backupsDir, backupIdentifier) |
|
||||
|
|
||||
ctx.attachment(backupIdentifier) |
ctx.attachment(backupIdentifier) |
||||
ctx.body = fs.createReadStream(backupFile) |
ctx.body = await performBackup(appId, backupIdentifier) |
||||
} |
} |
||||
|
|||||
@ -1,44 +1,37 @@ |
|||||
const CouchDB = require("../../db") |
const CouchDB = require("../../db") |
||||
const { resolve, join } = require("../../utilities/centralPath") |
const { join } = require("../../utilities/centralPath") |
||||
const { |
const { budibaseTempDir } = require("../../utilities/budibaseDir") |
||||
budibaseTempDir, |
const fileSystem = require("../../utilities/fileSystem") |
||||
budibaseAppsDir, |
const env = require("../../environment") |
||||
} = require("../../utilities/budibaseDir") |
|
||||
|
|
||||
exports.fetchAppComponentDefinitions = async function(ctx) { |
exports.fetchAppComponentDefinitions = async function(ctx) { |
||||
const appId = ctx.params.appId || ctx.appId |
const appId = ctx.params.appId || ctx.appId |
||||
const db = new CouchDB(appId) |
const db = new CouchDB(appId) |
||||
const app = await db.get(appId) |
const app = await db.get(appId) |
||||
|
|
||||
ctx.body = app.componentLibraries.reduce((acc, componentLibrary) => { |
let componentManifests = await Promise.all( |
||||
let appDirectory = resolve(budibaseAppsDir(), appId, "node_modules") |
app.componentLibraries.map(async library => { |
||||
|
let manifest |
||||
if (ctx.isDev) { |
if (env.isDev()) { |
||||
appDirectory = budibaseTempDir() |
manifest = require(join(budibaseTempDir(), library, "manifest.json")) |
||||
} |
} else { |
||||
|
manifest = await fileSystem.getComponentLibraryManifest(appId, library) |
||||
const componentJson = require(join( |
} |
||||
appDirectory, |
return { |
||||
componentLibrary, |
manifest, |
||||
ctx.isDev ? "" : "package", |
library, |
||||
"manifest.json" |
} |
||||
)) |
}) |
||||
|
) |
||||
const result = {} |
const definitions = {} |
||||
|
for (let { manifest, library } of componentManifests) { |
||||
// map over the components.json and add the library identifier as a key
|
for (let key of Object.keys(manifest)) { |
||||
// button -> @budibase/standard-components/button
|
const fullComponentName = `${library}/${key}`.toLowerCase() |
||||
for (let key of Object.keys(componentJson)) { |
definitions[fullComponentName] = { |
||||
const fullComponentName = `${componentLibrary}/${key}`.toLowerCase() |
|
||||
result[fullComponentName] = { |
|
||||
component: fullComponentName, |
component: fullComponentName, |
||||
...componentJson[key], |
...manifest[key], |
||||
} |
} |
||||
} |
} |
||||
|
} |
||||
return { |
ctx.body = definitions |
||||
...acc, |
|
||||
...result, |
|
||||
} |
|
||||
}, {}) |
|
||||
} |
} |
||||
|
|||||
@ -1,16 +0,0 @@ |
|||||
const setup = require("./utilities") |
|
||||
|
|
||||
describe("test things in the Cloud/Self hosted", () => { |
|
||||
describe("test self hosted static page", () => { |
|
||||
it("should be able to load the static page", async () => { |
|
||||
await setup.switchToCloudForFunction(async () => { |
|
||||
let request = setup.getRequest() |
|
||||
let config = setup.getConfig() |
|
||||
await config.init() |
|
||||
const res = await request.get(`/`).expect(200) |
|
||||
expect(res.text.includes("<title>Budibase self hosting️</title>")).toEqual(true) |
|
||||
setup.afterAll() |
|
||||
}) |
|
||||
}) |
|
||||
}) |
|
||||
}) |
|
||||
@ -0,0 +1,38 @@ |
|||||
|
const CouchDB = require("./index") |
||||
|
const { StaticDatabases } = require("./utils") |
||||
|
const env = require("../environment") |
||||
|
|
||||
|
const SELF_HOST_ERR = "Unable to access builder DB/doc - not self hosted." |
||||
|
const BUILDER_DB = StaticDatabases.BUILDER |
||||
|
|
||||
|
/** |
||||
|
* This is the builder database, right now this is a single, static database |
||||
|
* that is present across the whole system and determines some core functionality |
||||
|
* for the builder (e.g. storage of API keys). This has been limited to self hosting |
||||
|
* as it doesn't make as much sense against the currently design Cloud system. |
||||
|
*/ |
||||
|
|
||||
|
exports.getBuilderMainDoc = async () => { |
||||
|
if (!env.SELF_HOSTED) { |
||||
|
throw SELF_HOST_ERR |
||||
|
} |
||||
|
const db = new CouchDB(BUILDER_DB.name) |
||||
|
try { |
||||
|
return await db.get(BUILDER_DB.baseDoc) |
||||
|
} catch (err) { |
||||
|
// doesn't exist yet, nothing to get
|
||||
|
return { |
||||
|
_id: BUILDER_DB.baseDoc, |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
exports.setBuilderMainDoc = async doc => { |
||||
|
if (!env.SELF_HOSTED) { |
||||
|
throw SELF_HOST_ERR |
||||
|
} |
||||
|
// make sure to override the ID
|
||||
|
doc._id = BUILDER_DB.baseDoc |
||||
|
const db = new CouchDB(BUILDER_DB.name) |
||||
|
return db.put(doc) |
||||
|
} |
||||
@ -1,45 +1,64 @@ |
|||||
const { resolve, join } = require("./utilities/centralPath") |
function isTest() { |
||||
const { homedir } = require("os") |
return ( |
||||
const { app } = require("electron") |
process.env.NODE_ENV === "jest" || |
||||
|
process.env.NODE_ENV === "cypress" || |
||||
|
process.env.JEST_WORKER_ID != null |
||||
|
) |
||||
|
} |
||||
|
|
||||
let LOADED = false |
function isDev() { |
||||
|
return ( |
||||
|
process.env.NODE_ENV !== "production" && |
||||
|
process.env.BUDIBASE_ENVIRONMENT !== "production" |
||||
|
) |
||||
|
} |
||||
|
|
||||
if (!LOADED) { |
let LOADED = false |
||||
const homeDir = app ? app.getPath("home") : homedir() |
if (!LOADED && isDev() && !isTest()) { |
||||
const budibaseDir = join(homeDir, ".budibase") |
require("dotenv").config() |
||||
process.env.BUDIBASE_DIR = budibaseDir |
|
||||
require("dotenv").config({ path: resolve(budibaseDir, ".env") }) |
|
||||
LOADED = true |
LOADED = true |
||||
} |
} |
||||
|
|
||||
module.exports = { |
module.exports = { |
||||
CLIENT_ID: process.env.CLIENT_ID, |
// important
|
||||
NODE_ENV: process.env.NODE_ENV, |
|
||||
JWT_SECRET: process.env.JWT_SECRET, |
|
||||
BUDIBASE_DIR: process.env.BUDIBASE_DIR, |
|
||||
PORT: process.env.PORT, |
PORT: process.env.PORT, |
||||
|
JWT_SECRET: process.env.JWT_SECRET, |
||||
COUCH_DB_URL: process.env.COUCH_DB_URL, |
COUCH_DB_URL: process.env.COUCH_DB_URL, |
||||
|
MINIO_URL: process.env.MINIO_URL, |
||||
|
WORKER_URL: process.env.WORKER_URL, |
||||
|
SELF_HOSTED: process.env.SELF_HOSTED, |
||||
|
AWS_REGION: process.env.AWS_REGION, |
||||
|
ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS, |
||||
|
MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY, |
||||
|
MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY, |
||||
|
USE_QUOTAS: process.env.USE_QUOTAS, |
||||
|
// environment
|
||||
|
NODE_ENV: process.env.NODE_ENV, |
||||
|
JEST_WORKER_ID: process.env.JEST_WORKER_ID, |
||||
|
BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT, |
||||
|
// minor
|
||||
SALT_ROUNDS: process.env.SALT_ROUNDS, |
SALT_ROUNDS: process.env.SALT_ROUNDS, |
||||
LOGGER: process.env.LOGGER, |
LOGGER: process.env.LOGGER, |
||||
LOG_LEVEL: process.env.LOG_LEVEL, |
LOG_LEVEL: process.env.LOG_LEVEL, |
||||
AUTOMATION_DIRECTORY: process.env.AUTOMATION_DIRECTORY, |
AUTOMATION_DIRECTORY: process.env.AUTOMATION_DIRECTORY, |
||||
AUTOMATION_BUCKET: process.env.AUTOMATION_BUCKET, |
AUTOMATION_BUCKET: process.env.AUTOMATION_BUCKET, |
||||
BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT, |
|
||||
SENDGRID_API_KEY: process.env.SENDGRID_API_KEY, |
SENDGRID_API_KEY: process.env.SENDGRID_API_KEY, |
||||
CLOUD: process.env.CLOUD, |
|
||||
SELF_HOSTED: process.env.SELF_HOSTED, |
|
||||
WORKER_URL: process.env.WORKER_URL, |
|
||||
HOSTING_KEY: process.env.HOSTING_KEY, |
|
||||
DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT, |
DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT, |
||||
AWS_REGION: process.env.AWS_REGION, |
// old - to remove
|
||||
DEPLOYMENT_CREDENTIALS_URL: process.env.DEPLOYMENT_CREDENTIALS_URL, |
CLIENT_ID: process.env.CLIENT_ID, |
||||
|
BUDIBASE_DIR: process.env.BUDIBASE_DIR, |
||||
|
DEPLOYMENT_DB_URL: process.env.DEPLOYMENT_DB_URL, |
||||
BUDIBASE_API_KEY: process.env.BUDIBASE_API_KEY, |
BUDIBASE_API_KEY: process.env.BUDIBASE_API_KEY, |
||||
USERID_API_KEY: process.env.USERID_API_KEY, |
USERID_API_KEY: process.env.USERID_API_KEY, |
||||
ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS, |
DEPLOYMENT_CREDENTIALS_URL: process.env.DEPLOYMENT_CREDENTIALS_URL, |
||||
DEPLOYMENT_DB_URL: process.env.DEPLOYMENT_DB_URL, |
HOSTING_KEY: process.env.HOSTING_KEY, |
||||
LOCAL_TEMPLATES: process.env.LOCAL_TEMPLATES, |
|
||||
_set(key, value) { |
_set(key, value) { |
||||
process.env[key] = value |
process.env[key] = value |
||||
module.exports[key] = value |
module.exports[key] = value |
||||
}, |
}, |
||||
|
isTest, |
||||
|
isDev, |
||||
|
isProd: () => { |
||||
|
return !isDev() |
||||
|
}, |
||||
} |
} |
||||
|
|||||
@ -1,35 +0,0 @@ |
|||||
const { ensureDir, constants, copyFile } = require("fs-extra") |
|
||||
const { join } = require("../centralPath") |
|
||||
const { budibaseAppsDir } = require("../budibaseDir") |
|
||||
|
|
||||
/** |
|
||||
* Compile all the non-db static web assets that are required for the running of |
|
||||
* a budibase application. This includes the JSON structure of the DOM and |
|
||||
* the client library, a script responsible for reading the JSON structure |
|
||||
* and rendering the application. |
|
||||
* @param {string} appId id of the application we want to compile static assets for |
|
||||
*/ |
|
||||
module.exports = async appId => { |
|
||||
const publicPath = join(budibaseAppsDir(), appId, "public") |
|
||||
await ensureDir(publicPath) |
|
||||
await copyClientLib(publicPath) |
|
||||
} |
|
||||
|
|
||||
/** |
|
||||
* Copy the budibase client library and sourcemap from NPM to <appId>/public/. |
|
||||
* The client library is then served as a static asset when the budibase application |
|
||||
* is running in preview or prod |
|
||||
* @param {String} publicPath - path to write the client library to |
|
||||
*/ |
|
||||
const copyClientLib = async publicPath => { |
|
||||
const sourcepath = require.resolve("@budibase/client") |
|
||||
const destPath = join(publicPath, "budibase-client.js") |
|
||||
|
|
||||
await copyFile(sourcepath, destPath, constants.COPYFILE_FICLONE) |
|
||||
|
|
||||
await copyFile( |
|
||||
sourcepath + ".map", |
|
||||
destPath + ".map", |
|
||||
constants.COPYFILE_FICLONE |
|
||||
) |
|
||||
} |
|
||||
@ -1,29 +0,0 @@ |
|||||
const stream = require("stream") |
|
||||
const fetch = require("node-fetch") |
|
||||
const tar = require("tar-fs") |
|
||||
const zlib = require("zlib") |
|
||||
const { promisify } = require("util") |
|
||||
const packageJson = require("../../package.json") |
|
||||
|
|
||||
const streamPipeline = promisify(stream.pipeline) |
|
||||
|
|
||||
// can't really test this due to the downloading nature of it, wouldn't be a great test case
|
|
||||
/* istanbul ignore next */ |
|
||||
exports.downloadExtractComponentLibraries = async appFolder => { |
|
||||
const LIBRARIES = ["standard-components"] |
|
||||
|
|
||||
// Need to download tarballs directly from NPM as our users may not have node on their machine
|
|
||||
for (let lib of LIBRARIES) { |
|
||||
// download tarball
|
|
||||
const registryUrl = `https://registry.npmjs.org/@budibase/${lib}/-/${lib}-${packageJson.version}.tgz` |
|
||||
const response = await fetch(registryUrl) |
|
||||
if (!response.ok) |
|
||||
throw new Error(`unexpected response ${response.statusText}`) |
|
||||
|
|
||||
await streamPipeline( |
|
||||
response.body, |
|
||||
zlib.Unzip(), |
|
||||
tar.extract(`${appFolder}/node_modules/@budibase/${lib}`) |
|
||||
) |
|
||||
} |
|
||||
} |
|
||||
@ -1,40 +0,0 @@ |
|||||
const { budibaseTempDir } = require("./budibaseDir") |
|
||||
const { isDev } = require("./index") |
|
||||
const fs = require("fs") |
|
||||
const { join } = require("path") |
|
||||
const { downloadTemplate } = require("./templates") |
|
||||
|
|
||||
/** |
|
||||
* The single stack system (Cloud and Builder) should not make use of the file system where possible, |
|
||||
* this file handles all of the file access for the system with the intention of limiting it all to one |
|
||||
* place. Keeping all of this logic in one place means that when we need to do file system access (like |
|
||||
* downloading a package or opening a temporary file) in can be done in way that we can confirm it shouldn't |
|
||||
* be done through an object store instead. |
|
||||
*/ |
|
||||
|
|
||||
/** |
|
||||
* Checks if the system is currently in development mode and if it is makes sure |
|
||||
* everything required to function is ready. |
|
||||
*/ |
|
||||
exports.checkDevelopmentEnvironment = () => { |
|
||||
if (isDev() && !fs.existsSync(budibaseTempDir())) { |
|
||||
console.error( |
|
||||
"Please run a build before attempting to run server independently to fill 'tmp' directory." |
|
||||
) |
|
||||
process.exit(-1) |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
/** |
|
||||
* This function manages temporary template files which are stored by Koa. |
|
||||
* @param {Object} template The template object retrieved from the Koa context object. |
|
||||
* @returns {Object} Returns an fs read stream which can be loaded into the database. |
|
||||
*/ |
|
||||
exports.getTemplateStream = async template => { |
|
||||
if (template.file) { |
|
||||
return fs.createReadStream(template.file.path) |
|
||||
} else { |
|
||||
const templatePath = await downloadTemplate(...template.key.split("/")) |
|
||||
return fs.createReadStream(join(templatePath, "db", "dump.txt")) |
|
||||
} |
|
||||
} |
|
||||
@ -0,0 +1,204 @@ |
|||||
|
const { budibaseTempDir } = require("../budibaseDir") |
||||
|
const { isDev } = require("../index") |
||||
|
const fs = require("fs") |
||||
|
const { join } = require("path") |
||||
|
const uuid = require("uuid/v4") |
||||
|
const CouchDB = require("../../db") |
||||
|
const { ObjectStoreBuckets } = require("../../constants") |
||||
|
const { |
||||
|
upload, |
||||
|
retrieve, |
||||
|
streamUpload, |
||||
|
deleteFolder, |
||||
|
downloadTarball, |
||||
|
} = require("./utilities") |
||||
|
const { downloadLibraries, newAppPublicPath } = require("./newApp") |
||||
|
const download = require("download") |
||||
|
const env = require("../../environment") |
||||
|
const { homedir } = require("os") |
||||
|
const fetch = require("node-fetch") |
||||
|
|
||||
|
const DEFAULT_AUTOMATION_BUCKET = |
||||
|
"https://prod-budi-automations.s3-eu-west-1.amazonaws.com" |
||||
|
const DEFAULT_AUTOMATION_DIRECTORY = ".budibase-automations" |
||||
|
|
||||
|
/** |
||||
|
* The single stack system (Cloud and Builder) should not make use of the file system where possible, |
||||
|
* this file handles all of the file access for the system with the intention of limiting it all to one |
||||
|
* place. Keeping all of this logic in one place means that when we need to do file system access (like |
||||
|
* downloading a package or opening a temporary file) in can be done in way that we can confirm it shouldn't |
||||
|
* be done through an object store instead. |
||||
|
*/ |
||||
|
|
||||
|
/** |
||||
|
* Upon first startup of instance there may not be everything we need in tmp directory, set it up. |
||||
|
*/ |
||||
|
exports.init = () => { |
||||
|
const tempDir = budibaseTempDir() |
||||
|
if (!fs.existsSync(tempDir)) { |
||||
|
fs.mkdirSync(tempDir) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* Checks if the system is currently in development mode and if it is makes sure |
||||
|
* everything required to function is ready. |
||||
|
*/ |
||||
|
exports.checkDevelopmentEnvironment = () => { |
||||
|
if (!isDev()) { |
||||
|
return |
||||
|
} |
||||
|
let error |
||||
|
if (!fs.existsSync(budibaseTempDir())) { |
||||
|
error = |
||||
|
"Please run a build before attempting to run server independently to fill 'tmp' directory." |
||||
|
} |
||||
|
if (!fs.existsSync(join(process.cwd(), ".env"))) { |
||||
|
error = "Must run via yarn once to generate environment." |
||||
|
} |
||||
|
if (error) { |
||||
|
console.error(error) |
||||
|
process.exit(-1) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* This function manages temporary template files which are stored by Koa. |
||||
|
* @param {Object} template The template object retrieved from the Koa context object. |
||||
|
* @returns {Object} Returns an fs read stream which can be loaded into the database. |
||||
|
*/ |
||||
|
exports.getTemplateStream = async template => { |
||||
|
if (template.file) { |
||||
|
return fs.createReadStream(template.file.path) |
||||
|
} else { |
||||
|
const tmpPath = await exports.downloadTemplate(...template.key.split("/")) |
||||
|
return fs.createReadStream(join(tmpPath, "db", "dump.txt")) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* Used to retrieve a handlebars file from the system which will be used as a template. |
||||
|
* This is allowable as the template handlebars files should be static and identical across |
||||
|
* the cluster. |
||||
|
* @param {string} path The path to the handlebars file which is to be loaded. |
||||
|
* @returns {string} The loaded handlebars file as a string - loaded as utf8. |
||||
|
*/ |
||||
|
exports.loadHandlebarsFile = path => { |
||||
|
return fs.readFileSync(path, "utf8") |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* When return a file from the API need to write the file to the system temporarily so we |
||||
|
* can create a read stream to send. |
||||
|
* @param {string} contents the contents of the file which is to be returned from the API. |
||||
|
* @return {Object} the read stream which can be put into the koa context body. |
||||
|
*/ |
||||
|
exports.apiFileReturn = contents => { |
||||
|
const path = join(budibaseTempDir(), uuid()) |
||||
|
fs.writeFileSync(path, contents) |
||||
|
return fs.createReadStream(path) |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* Takes a copy of the database state for an app to the object store. |
||||
|
* @param {string} appId The ID of the app which is to be backed up. |
||||
|
* @param {string} backupName The name of the backup located in the object store. |
||||
|
* @return The backup has been completed when this promise completes and returns a file stream |
||||
|
* to the temporary backup file (to return via API if required). |
||||
|
*/ |
||||
|
exports.performBackup = async (appId, backupName) => { |
||||
|
const path = join(budibaseTempDir(), backupName) |
||||
|
const writeStream = fs.createWriteStream(path) |
||||
|
// perform couch dump
|
||||
|
const instanceDb = new CouchDB(appId) |
||||
|
await instanceDb.dump(writeStream, {}) |
||||
|
// write the file to the object store
|
||||
|
await streamUpload( |
||||
|
ObjectStoreBuckets.BACKUPS, |
||||
|
join(appId, backupName), |
||||
|
fs.createReadStream(path) |
||||
|
) |
||||
|
return fs.createReadStream(path) |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* Downloads required libraries and creates a new path in the object store. |
||||
|
* @param {string} appId The ID of the app which is being created. |
||||
|
* @return {Promise<void>} once promise completes app resources should be ready in object store. |
||||
|
*/ |
||||
|
exports.createApp = async appId => { |
||||
|
await downloadLibraries(appId) |
||||
|
await newAppPublicPath(appId) |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* Removes all of the assets created for an app in the object store. |
||||
|
* @param {string} appId The ID of the app which is being deleted. |
||||
|
* @return {Promise<void>} once promise completes the app resources will be removed from object store. |
||||
|
*/ |
||||
|
exports.deleteApp = async appId => { |
||||
|
await deleteFolder(ObjectStoreBuckets.APPS, `${appId}/`) |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* Retrieves a template and pipes it to minio as well as making it available temporarily. |
||||
|
* @param {string} type The type of template which is to be retrieved. |
||||
|
* @param name |
||||
|
* @return {Promise<*>} |
||||
|
*/ |
||||
|
exports.downloadTemplate = async (type, name) => { |
||||
|
const DEFAULT_TEMPLATES_BUCKET = |
||||
|
"prod-budi-templates.s3-eu-west-1.amazonaws.com" |
||||
|
const templateUrl = `https://${DEFAULT_TEMPLATES_BUCKET}/templates/${type}/${name}.tar.gz` |
||||
|
return downloadTarball(templateUrl, ObjectStoreBuckets.TEMPLATES, type) |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* Retrieves component libraries from object store (or tmp symlink if in local) |
||||
|
*/ |
||||
|
exports.getComponentLibraryManifest = async (appId, library) => { |
||||
|
const path = join(appId, "node_modules", library, "package", "manifest.json") |
||||
|
let resp = await retrieve(ObjectStoreBuckets.APPS, path) |
||||
|
if (typeof resp !== "string") { |
||||
|
resp = resp.toString("utf8") |
||||
|
} |
||||
|
return JSON.parse(resp) |
||||
|
} |
||||
|
|
||||
|
exports.automationInit = async () => { |
||||
|
const directory = |
||||
|
env.AUTOMATION_DIRECTORY || join(homedir(), DEFAULT_AUTOMATION_DIRECTORY) |
||||
|
const bucket = env.AUTOMATION_BUCKET || DEFAULT_AUTOMATION_BUCKET |
||||
|
if (!fs.existsSync(directory)) { |
||||
|
fs.mkdirSync(directory, { recursive: true }) |
||||
|
} |
||||
|
// env setup to get async packages
|
||||
|
let response = await fetch(`${bucket}/manifest.json`) |
||||
|
return response.json() |
||||
|
} |
||||
|
|
||||
|
exports.getExternalAutomationStep = async (name, version, bundleName) => { |
||||
|
const directory = |
||||
|
env.AUTOMATION_DIRECTORY || join(homedir(), DEFAULT_AUTOMATION_DIRECTORY) |
||||
|
const bucket = env.AUTOMATION_BUCKET || DEFAULT_AUTOMATION_BUCKET |
||||
|
try { |
||||
|
return require(join(directory, bundleName)) |
||||
|
} catch (err) { |
||||
|
await download(`${bucket}/${name}/${version}/${bundleName}`, directory) |
||||
|
return require(join(directory, bundleName)) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* All file reads come through here just to make sure all of them make sense |
||||
|
* allows a centralised location to check logic is all good. |
||||
|
*/ |
||||
|
exports.readFileSync = (filepath, options = "utf8") => { |
||||
|
return fs.readFileSync(filepath, options) |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* Full function definition for below can be found in the utilities. |
||||
|
*/ |
||||
|
exports.upload = upload |
||||
|
exports.retrieve = retrieve |
||||
@ -0,0 +1,34 @@ |
|||||
|
const packageJson = require("../../../package.json") |
||||
|
const { join } = require("path") |
||||
|
const { ObjectStoreBuckets } = require("../../constants") |
||||
|
const { streamUpload, downloadTarball } = require("./utilities") |
||||
|
const fs = require("fs") |
||||
|
|
||||
|
const BUCKET_NAME = ObjectStoreBuckets.APPS |
||||
|
|
||||
|
// can't really test this due to the downloading nature of it, wouldn't be a great test case
|
||||
|
/* istanbul ignore next */ |
||||
|
exports.downloadLibraries = async appId => { |
||||
|
const LIBRARIES = ["standard-components"] |
||||
|
|
||||
|
// Need to download tarballs directly from NPM as our users may not have node on their machine
|
||||
|
for (let lib of LIBRARIES) { |
||||
|
// download tarball
|
||||
|
const registryUrl = `https://registry.npmjs.org/@budibase/${lib}/-/${lib}-${packageJson.version}.tgz` |
||||
|
const path = join(appId, "node_modules", "@budibase", lib) |
||||
|
await downloadTarball(registryUrl, BUCKET_NAME, path) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
exports.newAppPublicPath = async appId => { |
||||
|
const path = join(appId, "public") |
||||
|
const sourcepath = require.resolve("@budibase/client") |
||||
|
const destPath = join(path, "budibase-client.js") |
||||
|
|
||||
|
await streamUpload(BUCKET_NAME, destPath, fs.createReadStream(sourcepath)) |
||||
|
await streamUpload( |
||||
|
BUCKET_NAME, |
||||
|
destPath + ".map", |
||||
|
fs.createReadStream(sourcepath + ".map") |
||||
|
) |
||||
|
} |
||||
@ -1,25 +1,20 @@ |
|||||
const fs = require("fs") |
|
||||
const jimp = require("jimp") |
const jimp = require("jimp") |
||||
const fsPromises = fs.promises |
|
||||
|
|
||||
const FORMATS = { |
const FORMATS = { |
||||
IMAGES: ["png", "jpg", "jpeg", "gif", "bmp", "tiff"], |
IMAGES: ["png", "jpg", "jpeg", "gif", "bmp", "tiff"], |
||||
} |
} |
||||
|
|
||||
function processImage(file) { |
function processImage(file) { |
||||
|
// this will overwrite the temp file
|
||||
return jimp.read(file.path).then(img => { |
return jimp.read(file.path).then(img => { |
||||
return img.resize(300, jimp.AUTO).write(file.outputPath) |
return img.resize(300, jimp.AUTO).write(file.path) |
||||
}) |
}) |
||||
} |
} |
||||
|
|
||||
async function process(file) { |
async function process(file) { |
||||
if (FORMATS.IMAGES.includes(file.extension.toLowerCase())) { |
if (FORMATS.IMAGES.includes(file.extension.toLowerCase())) { |
||||
await processImage(file) |
await processImage(file) |
||||
return file |
|
||||
} |
} |
||||
|
|
||||
// No processing required
|
|
||||
await fsPromises.copyFile(file.path, file.outputPath) |
|
||||
return file |
return file |
||||
} |
} |
||||
|
|
||||
@ -0,0 +1,221 @@ |
|||||
|
const sanitize = require("sanitize-s3-objectkey") |
||||
|
const AWS = require("aws-sdk") |
||||
|
const stream = require("stream") |
||||
|
const fetch = require("node-fetch") |
||||
|
const tar = require("tar-fs") |
||||
|
const zlib = require("zlib") |
||||
|
const { promisify } = require("util") |
||||
|
const { join } = require("path") |
||||
|
const fs = require("fs") |
||||
|
const { budibaseTempDir } = require("../budibaseDir") |
||||
|
const env = require("../../environment") |
||||
|
const { ObjectStoreBuckets } = require("../../constants") |
||||
|
|
||||
|
const streamPipeline = promisify(stream.pipeline) |
||||
|
|
||||
|
const CONTENT_TYPE_MAP = { |
||||
|
html: "text/html", |
||||
|
css: "text/css", |
||||
|
js: "application/javascript", |
||||
|
} |
||||
|
const STRING_CONTENT_TYPES = [ |
||||
|
CONTENT_TYPE_MAP.html, |
||||
|
CONTENT_TYPE_MAP.css, |
||||
|
CONTENT_TYPE_MAP.js, |
||||
|
] |
||||
|
|
||||
|
function publicPolicy(bucketName) { |
||||
|
return { |
||||
|
Version: "2012-10-17", |
||||
|
Statement: [ |
||||
|
{ |
||||
|
Effect: "Allow", |
||||
|
Principal: { |
||||
|
AWS: ["*"], |
||||
|
}, |
||||
|
Action: "s3:GetObject", |
||||
|
Resource: [`arn:aws:s3:::${bucketName}/*`], |
||||
|
}, |
||||
|
], |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
const PUBLIC_BUCKETS = [ObjectStoreBuckets.APPS] |
||||
|
|
||||
|
/** |
||||
|
* Gets a connection to the object store using the S3 SDK. |
||||
|
* @param {string} bucket the name of the bucket which blobs will be uploaded/retrieved from. |
||||
|
* @return {Object} an S3 object store object, check S3 Nodejs SDK for usage. |
||||
|
* @constructor |
||||
|
*/ |
||||
|
exports.ObjectStore = bucket => { |
||||
|
if (env.SELF_HOSTED) { |
||||
|
AWS.config.update({ |
||||
|
accessKeyId: env.MINIO_ACCESS_KEY, |
||||
|
secretAccessKey: env.MINIO_SECRET_KEY, |
||||
|
}) |
||||
|
} |
||||
|
const config = { |
||||
|
s3ForcePathStyle: true, |
||||
|
signatureVersion: "v4", |
||||
|
params: { |
||||
|
Bucket: bucket, |
||||
|
}, |
||||
|
} |
||||
|
if (env.MINIO_URL) { |
||||
|
config.endpoint = env.MINIO_URL |
||||
|
} |
||||
|
return new AWS.S3(config) |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* Given an object store and a bucket name this will make sure the bucket exists, |
||||
|
* if it does not exist then it will create it. |
||||
|
*/ |
||||
|
exports.makeSureBucketExists = async (client, bucketName) => { |
||||
|
try { |
||||
|
await client |
||||
|
.headBucket({ |
||||
|
Bucket: bucketName, |
||||
|
}) |
||||
|
.promise() |
||||
|
} catch (err) { |
||||
|
// bucket doesn't exist create it
|
||||
|
if (err.statusCode === 404) { |
||||
|
await client |
||||
|
.createBucket({ |
||||
|
Bucket: bucketName, |
||||
|
}) |
||||
|
.promise() |
||||
|
// public buckets are quite hidden in the system, make sure
|
||||
|
// no bucket is set accidentally
|
||||
|
if (PUBLIC_BUCKETS.includes(bucketName)) { |
||||
|
await client |
||||
|
.putBucketPolicy({ |
||||
|
Bucket: bucketName, |
||||
|
Policy: JSON.stringify(publicPolicy(bucketName)), |
||||
|
}) |
||||
|
.promise() |
||||
|
} |
||||
|
} else { |
||||
|
throw err |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* Uploads the contents of a file given the required parameters, useful when |
||||
|
* temp files in use (for example file uploaded as an attachment). |
||||
|
*/ |
||||
|
exports.upload = async ({ bucket, filename, path, type, metadata }) => { |
||||
|
const extension = [...filename.split(".")].pop() |
||||
|
const fileBytes = fs.readFileSync(path) |
||||
|
|
||||
|
const objectStore = exports.ObjectStore(bucket) |
||||
|
await exports.makeSureBucketExists(objectStore, bucket) |
||||
|
|
||||
|
const config = { |
||||
|
// windows file paths need to be converted to forward slashes for s3
|
||||
|
Key: sanitize(filename).replace(/\\/g, "/"), |
||||
|
Body: fileBytes, |
||||
|
ContentType: type || CONTENT_TYPE_MAP[extension.toLowerCase()], |
||||
|
} |
||||
|
if (metadata) { |
||||
|
config.Metadata = metadata |
||||
|
} |
||||
|
return objectStore.upload(config).promise() |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* Similar to the upload function but can be used to send a file stream |
||||
|
* through to the object store. |
||||
|
*/ |
||||
|
exports.streamUpload = async (bucket, filename, stream) => { |
||||
|
const objectStore = exports.ObjectStore(bucket) |
||||
|
await exports.makeSureBucketExists(objectStore, bucket) |
||||
|
|
||||
|
const params = { |
||||
|
Bucket: bucket, |
||||
|
Key: sanitize(filename).replace(/\\/g, "/"), |
||||
|
Body: stream, |
||||
|
} |
||||
|
return objectStore.upload(params).promise() |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* retrieves the contents of a file from the object store, if it is a known content type it |
||||
|
* will be converted, otherwise it will be returned as a buffer stream. |
||||
|
*/ |
||||
|
exports.retrieve = async (bucket, filename) => { |
||||
|
const objectStore = exports.ObjectStore(bucket) |
||||
|
const params = { |
||||
|
Bucket: bucket, |
||||
|
Key: sanitize(filename).replace(/\\/g, "/"), |
||||
|
} |
||||
|
const response = await objectStore.getObject(params).promise() |
||||
|
// currently these are all strings
|
||||
|
if (STRING_CONTENT_TYPES.includes(response.ContentType)) { |
||||
|
return response.Body.toString("utf8") |
||||
|
} else { |
||||
|
return response.Body |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
exports.deleteFolder = async (bucket, folder) => { |
||||
|
const client = exports.ObjectStore(bucket) |
||||
|
const listParams = { |
||||
|
Bucket: bucket, |
||||
|
Prefix: folder, |
||||
|
} |
||||
|
|
||||
|
let response = await client.listObjects(listParams).promise() |
||||
|
if (response.Contents.length === 0) { |
||||
|
return |
||||
|
} |
||||
|
const deleteParams = { |
||||
|
Bucket: bucket, |
||||
|
Delete: { |
||||
|
Objects: [], |
||||
|
}, |
||||
|
} |
||||
|
|
||||
|
response.Contents.forEach(content => { |
||||
|
deleteParams.Delete.Objects.push({ Key: content.Key }) |
||||
|
}) |
||||
|
|
||||
|
response = await client.deleteObjects(deleteParams).promise() |
||||
|
// can only empty 1000 items at once
|
||||
|
if (response.Deleted.length === 1000) { |
||||
|
return exports.deleteFolder(bucket, folder) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
exports.uploadDirectory = async (bucket, localPath, bucketPath) => { |
||||
|
let uploads = [] |
||||
|
const files = fs.readdirSync(localPath, { withFileTypes: true }) |
||||
|
for (let file of files) { |
||||
|
const path = join(bucketPath, file.name) |
||||
|
const local = join(localPath, file.name) |
||||
|
if (file.isDirectory()) { |
||||
|
uploads.push(exports.uploadDirectory(bucket, local, path)) |
||||
|
} else { |
||||
|
uploads.push( |
||||
|
exports.streamUpload(bucket, path, fs.createReadStream(local)) |
||||
|
) |
||||
|
} |
||||
|
} |
||||
|
await Promise.all(uploads) |
||||
|
} |
||||
|
|
||||
|
exports.downloadTarball = async (url, bucket, path) => { |
||||
|
const response = await fetch(url) |
||||
|
if (!response.ok) { |
||||
|
throw new Error(`unexpected response ${response.statusText}`) |
||||
|
} |
||||
|
|
||||
|
const tmpPath = join(budibaseTempDir(), path) |
||||
|
await streamPipeline(response.body, zlib.Unzip(), tar.extract(tmpPath)) |
||||
|
await exports.uploadDirectory(bucket, tmpPath, path) |
||||
|
// return the temporary path incase there is a use for it
|
||||
|
return tmpPath |
||||
|
} |
||||
@ -1,81 +0,0 @@ |
|||||
const fs = require("fs-extra") |
|
||||
const { join } = require("./centralPath") |
|
||||
const os = require("os") |
|
||||
const fetch = require("node-fetch") |
|
||||
const stream = require("stream") |
|
||||
const tar = require("tar-fs") |
|
||||
const zlib = require("zlib") |
|
||||
const { promisify } = require("util") |
|
||||
const streamPipeline = promisify(stream.pipeline) |
|
||||
const { budibaseAppsDir } = require("./budibaseDir") |
|
||||
const env = require("../environment") |
|
||||
const CouchDB = require("../db") |
|
||||
|
|
||||
const DEFAULT_TEMPLATES_BUCKET = |
|
||||
"prod-budi-templates.s3-eu-west-1.amazonaws.com" |
|
||||
|
|
||||
exports.getLocalTemplates = function() { |
|
||||
const templatesDir = join(os.homedir(), ".budibase", "templates", "app") |
|
||||
const templateObj = { app: {} } |
|
||||
fs.ensureDirSync(templatesDir) |
|
||||
const templateNames = fs.readdirSync(templatesDir) |
|
||||
for (let name of templateNames) { |
|
||||
templateObj.app[name] = { |
|
||||
name, |
|
||||
category: "local", |
|
||||
description: "local template", |
|
||||
type: "app", |
|
||||
key: `app/${name}`, |
|
||||
} |
|
||||
} |
|
||||
return templateObj |
|
||||
} |
|
||||
|
|
||||
// can't really test this, downloading is just not something we should do in a behavioural test
|
|
||||
/* istanbul ignore next */ |
|
||||
exports.downloadTemplate = async function(type, name) { |
|
||||
const dirName = join(budibaseAppsDir(), "templates", type, name) |
|
||||
if (env.LOCAL_TEMPLATES) { |
|
||||
return dirName |
|
||||
} |
|
||||
const templateUrl = `https://${DEFAULT_TEMPLATES_BUCKET}/templates/${type}/${name}.tar.gz` |
|
||||
const response = await fetch(templateUrl) |
|
||||
|
|
||||
if (!response.ok) { |
|
||||
throw new Error( |
|
||||
`Error downloading template ${type}:${name}: ${response.statusText}` |
|
||||
) |
|
||||
} |
|
||||
|
|
||||
// stream the response, unzip and extract
|
|
||||
await streamPipeline( |
|
||||
response.body, |
|
||||
zlib.Unzip(), |
|
||||
tar.extract(join(budibaseAppsDir(), "templates", type)) |
|
||||
) |
|
||||
|
|
||||
return dirName |
|
||||
} |
|
||||
|
|
||||
async function performDump({ dir, appId, name = "dump.txt" }) { |
|
||||
const writeStream = fs.createWriteStream(join(dir, name)) |
|
||||
// perform couch dump
|
|
||||
const instanceDb = new CouchDB(appId) |
|
||||
await instanceDb.dump(writeStream, {}) |
|
||||
} |
|
||||
|
|
||||
exports.performDump = performDump |
|
||||
|
|
||||
exports.exportTemplateFromApp = async function({ templateName, appId }) { |
|
||||
// Copy frontend files
|
|
||||
const templatesDir = join( |
|
||||
budibaseAppsDir(), |
|
||||
"templates", |
|
||||
"app", |
|
||||
templateName, |
|
||||
"db" |
|
||||
) |
|
||||
fs.ensureDirSync(templatesDir) |
|
||||
await performDump({ dir: templatesDir, appId }) |
|
||||
return templatesDir |
|
||||
} |
|
||||
Loading…
Reference in new issue