mirror of https://github.com/Budibase/budibase.git
Michael Drury
5 years ago
committed by
GitHub
GitHub
90 changed files with 2510 additions and 1343 deletions
@ -1,16 +0,0 @@ |
|||
version: "3" |
|||
|
|||
services: |
|||
app-service: |
|||
build: ./server |
|||
volumes: |
|||
- ./server:/app |
|||
environment: |
|||
SELF_HOSTED: 1 |
|||
PORT: 4002 |
|||
|
|||
worker-service: |
|||
build: ./worker |
|||
environment: |
|||
SELF_HOSTED: 1, |
|||
PORT: 4003 |
|||
@ -1 +0,0 @@ |
|||
../../packages/server/ |
|||
@ -1 +0,0 @@ |
|||
../../packages/worker/ |
|||
@ -0,0 +1,76 @@ |
|||
version: "3" |
|||
|
|||
# optional ports are specified throughout for more advanced use cases. |
|||
|
|||
services: |
|||
minio-service: |
|||
container_name: budi-minio-dev |
|||
restart: always |
|||
image: minio/minio |
|||
volumes: |
|||
- minio_data:/data |
|||
ports: |
|||
- "${MINIO_PORT}:9000" |
|||
environment: |
|||
MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} |
|||
MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} |
|||
MINIO_BROWSER: "off" |
|||
command: server /data |
|||
healthcheck: |
|||
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] |
|||
interval: 30s |
|||
timeout: 20s |
|||
retries: 3 |
|||
|
|||
proxy-service: |
|||
container_name: budi-envoy-dev |
|||
restart: always |
|||
image: envoyproxy/envoy:v1.16-latest |
|||
volumes: |
|||
- ./envoy.dev.yaml:/etc/envoy/envoy.yaml |
|||
ports: |
|||
- "${MAIN_PORT}:10000" |
|||
#- "9901:9901" |
|||
depends_on: |
|||
- minio-service |
|||
- couchdb-service |
|||
|
|||
couchdb-service: |
|||
container_name: budi-couchdb-dev |
|||
restart: always |
|||
image: apache/couchdb:3.0 |
|||
environment: |
|||
- COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} |
|||
- COUCHDB_USER=${COUCH_DB_USER} |
|||
ports: |
|||
- "${COUCH_DB_PORT}:5984" |
|||
#- "4369:4369" |
|||
#- "9100:9100" |
|||
volumes: |
|||
- couchdb_data:/opt/couchdb/data |
|||
|
|||
couch-init: |
|||
image: curlimages/curl |
|||
environment: |
|||
PUT_CALL: "curl -u ${COUCH_DB_USER}:${COUCH_DB_PASSWORD} -X PUT couchdb-service:5984" |
|||
depends_on: |
|||
- couchdb-service |
|||
command: ["sh","-c","sleep 10 && $${PUT_CALL}/_users && $${PUT_CALL}/_replicator; fg;"] |
|||
|
|||
redis-service: |
|||
container_name: budi-redis-dev |
|||
restart: always |
|||
image: redis |
|||
ports: |
|||
- "${REDIS_PORT}:6379" |
|||
volumes: |
|||
- redis_data:/data |
|||
|
|||
|
|||
volumes: |
|||
couchdb_data: |
|||
driver: local |
|||
minio_data: |
|||
driver: local |
|||
redis_data: |
|||
driver: local |
|||
@ -0,0 +1,79 @@ |
|||
static_resources: |
|||
listeners: |
|||
- name: main_listener |
|||
address: |
|||
socket_address: { address: 0.0.0.0, port_value: 10000 } |
|||
filter_chains: |
|||
- filters: |
|||
- name: envoy.filters.network.http_connection_manager |
|||
typed_config: |
|||
"@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager |
|||
stat_prefix: ingress |
|||
codec_type: auto |
|||
route_config: |
|||
name: local_route |
|||
virtual_hosts: |
|||
- name: local_services |
|||
domains: ["*"] |
|||
routes: |
|||
- match: { prefix: "/db/" } |
|||
route: |
|||
cluster: couchdb-service |
|||
prefix_rewrite: "/" |
|||
|
|||
- match: { prefix: "/cache/" } |
|||
route: |
|||
cluster: redis-service |
|||
prefix_rewrite: "/" |
|||
|
|||
# minio is on the default route because this works |
|||
# best, minio + AWS SDK doesn't handle path proxy |
|||
- match: { prefix: "/" } |
|||
route: |
|||
cluster: minio-service |
|||
|
|||
http_filters: |
|||
- name: envoy.filters.http.router |
|||
|
|||
clusters: |
|||
- name: minio-service |
|||
connect_timeout: 0.25s |
|||
type: strict_dns |
|||
lb_policy: round_robin |
|||
load_assignment: |
|||
cluster_name: minio-service |
|||
endpoints: |
|||
- lb_endpoints: |
|||
- endpoint: |
|||
address: |
|||
socket_address: |
|||
address: minio-service |
|||
port_value: 9000 |
|||
|
|||
- name: couchdb-service |
|||
connect_timeout: 0.25s |
|||
type: strict_dns |
|||
lb_policy: round_robin |
|||
load_assignment: |
|||
cluster_name: couchdb-service |
|||
endpoints: |
|||
- lb_endpoints: |
|||
- endpoint: |
|||
address: |
|||
socket_address: |
|||
address: couchdb-service |
|||
port_value: 5984 |
|||
|
|||
- name: redis-service |
|||
connect_timeout: 0.25s |
|||
type: strict_dns |
|||
lb_policy: round_robin |
|||
load_assignment: |
|||
cluster_name: redis-service |
|||
endpoints: |
|||
- lb_endpoints: |
|||
- endpoint: |
|||
address: |
|||
socket_address: |
|||
address: redis-service |
|||
port_value: 6379 |
|||
@ -1,17 +0,0 @@ |
|||
# url of couch db, including username and password |
|||
# http://admin:password@localhost:5984 |
|||
COUCH_DB_URL={{couchDbUrl}} |
|||
|
|||
# identifies a client database - i.e. group of apps |
|||
CLIENT_ID={{clientId}} |
|||
|
|||
# used to create cookie hashes |
|||
JWT_SECRET={{cookieKey1}} |
|||
|
|||
# error level for koa-pino |
|||
LOG_LEVEL=info |
|||
|
|||
DEPLOYMENT_CREDENTIALS_URL="https://dt4mpwwap8.execute-api.eu-west-1.amazonaws.com/prod/" |
|||
DEPLOYMENT_DB_URL="https://couchdb.budi.live:5984" |
|||
SENTRY_DSN=https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131 |
|||
ENABLE_ANALYTICS="true" |
|||
@ -0,0 +1,96 @@ |
|||
#!/usr/bin/env node
|
|||
const compose = require("docker-compose") |
|||
const path = require("path") |
|||
const fs = require("fs") |
|||
|
|||
// This script wraps docker-compose allowing you to manage your dev infrastructure with simple commands.
|
|||
const CONFIG = { |
|||
cwd: path.resolve(process.cwd(), "../../hosting"), |
|||
config: "docker-compose.dev.yaml", |
|||
log: true, |
|||
} |
|||
|
|||
const Commands = { |
|||
Up: "up", |
|||
Down: "down", |
|||
Nuke: "nuke", |
|||
} |
|||
|
|||
async function init() { |
|||
const envFilePath = path.join(process.cwd(), ".env") |
|||
if (fs.existsSync(envFilePath)) { |
|||
return |
|||
} |
|||
const envFileJson = { |
|||
PORT: 4001, |
|||
MINIO_URL: "http://localhost:10000/", |
|||
COUCH_DB_URL: "http://budibase:budibase@localhost:10000/db/", |
|||
WORKER_URL: "http://localhost:4002", |
|||
JWT_SECRET: "testsecret", |
|||
MINIO_ACCESS_KEY: "budibase", |
|||
MINIO_SECRET_KEY: "budibase", |
|||
COUCH_DB_PASSWORD: "budibase", |
|||
COUCH_DB_USER: "budibase", |
|||
SELF_HOSTED: 1, |
|||
} |
|||
let envFile = "" |
|||
Object.keys(envFileJson).forEach(key => { |
|||
envFile += `${key}=${envFileJson[key]}\n` |
|||
}) |
|||
fs.writeFileSync(envFilePath, envFile) |
|||
} |
|||
|
|||
async function up() { |
|||
console.log("Spinning up your budibase dev environment... 🔧✨") |
|||
await init() |
|||
await compose.upAll(CONFIG) |
|||
} |
|||
|
|||
async function down() { |
|||
console.log("Spinning down your budibase dev environment... 🌇") |
|||
await compose.stop(CONFIG) |
|||
} |
|||
|
|||
async function nuke() { |
|||
console.log( |
|||
"Clearing down your budibase dev environment, including all containers and volumes... 💥" |
|||
) |
|||
await compose.down(CONFIG) |
|||
} |
|||
|
|||
const managementCommand = process.argv.slice(2)[0] |
|||
|
|||
if ( |
|||
!managementCommand || |
|||
!Object.values(Commands).some(command => managementCommand === command) |
|||
) { |
|||
throw new Error( |
|||
"You must supply either an 'up', 'down' or 'nuke' commmand to manage the budibase development environment." |
|||
) |
|||
} |
|||
|
|||
let command |
|||
switch (managementCommand) { |
|||
case Commands.Up: |
|||
command = up |
|||
break |
|||
case Commands.Down: |
|||
command = down |
|||
break |
|||
case Commands.Nuke: |
|||
command = nuke |
|||
break |
|||
default: |
|||
command = up |
|||
} |
|||
|
|||
command() |
|||
.then(() => { |
|||
console.log("Done! 🎉") |
|||
}) |
|||
.catch(err => { |
|||
console.error( |
|||
"Something went wrong while managing budibase dev environment:", |
|||
err.message |
|||
) |
|||
}) |
|||
@ -1,17 +0,0 @@ |
|||
const { join } = require("path") |
|||
const { homedir } = require("os") |
|||
|
|||
const initialiseBudibase = require("../src/utilities/initialiseBudibase") |
|||
const DIRECTORY = "~/.budibase" |
|||
|
|||
function run() { |
|||
let opts = {} |
|||
let dir = DIRECTORY |
|||
opts.quiet = true |
|||
opts.dir = dir.startsWith("~") ? join(homedir(), dir.substring(1)) : dir |
|||
return initialiseBudibase(opts) |
|||
} |
|||
|
|||
run().then(() => { |
|||
console.log("Init complete.") |
|||
}) |
|||
@ -1,56 +1,32 @@ |
|||
const fs = require("fs") |
|||
const { join } = require("../../utilities/centralPath") |
|||
const readline = require("readline") |
|||
const { budibaseAppsDir } = require("../../utilities/budibaseDir") |
|||
const env = require("../../environment") |
|||
const ENV_FILE_PATH = "/.env" |
|||
const builderDB = require("../../db/builder") |
|||
|
|||
exports.fetch = async function(ctx) { |
|||
ctx.status = 200 |
|||
ctx.body = { |
|||
budibase: env.BUDIBASE_API_KEY, |
|||
userId: env.USERID_API_KEY, |
|||
try { |
|||
const mainDoc = await builderDB.getBuilderMainDoc() |
|||
ctx.body = mainDoc.apiKeys ? mainDoc.apiKeys : {} |
|||
} catch (err) { |
|||
/* istanbul ignore next */ |
|||
ctx.throw(400, err) |
|||
} |
|||
} |
|||
|
|||
exports.update = async function(ctx) { |
|||
const key = `${ctx.params.key.toUpperCase()}_API_KEY` |
|||
const key = ctx.params.key |
|||
const value = ctx.request.body.value |
|||
|
|||
// set environment variables
|
|||
env._set(key, value) |
|||
|
|||
// Write to file
|
|||
await updateValues([key, value]) |
|||
|
|||
ctx.status = 200 |
|||
ctx.message = `Updated ${ctx.params.key} API key succesfully.` |
|||
ctx.body = { [ctx.params.key]: ctx.request.body.value } |
|||
} |
|||
|
|||
async function updateValues([key, value]) { |
|||
let newContent = "" |
|||
let keyExists = false |
|||
let envPath = join(budibaseAppsDir(), ENV_FILE_PATH) |
|||
const readInterface = readline.createInterface({ |
|||
input: fs.createReadStream(envPath), |
|||
output: process.stdout, |
|||
console: false, |
|||
}) |
|||
readInterface.on("line", function(line) { |
|||
// Mutate lines and change API Key
|
|||
if (line.startsWith(key)) { |
|||
line = `${key}=${value}` |
|||
keyExists = true |
|||
try { |
|||
const mainDoc = await builderDB.getBuilderMainDoc() |
|||
if (mainDoc.apiKeys == null) { |
|||
mainDoc.apiKeys = {} |
|||
} |
|||
newContent = `${newContent}\n${line}` |
|||
}) |
|||
readInterface.on("close", function() { |
|||
// Write file here
|
|||
if (!keyExists) { |
|||
// Add API Key if it doesn't exist in the file at all
|
|||
newContent = `${newContent}\n${key}=${value}` |
|||
mainDoc.apiKeys[key] = value |
|||
const resp = await builderDB.setBuilderMainDoc(mainDoc) |
|||
ctx.body = { |
|||
_id: resp.id, |
|||
_rev: resp.rev, |
|||
} |
|||
fs.writeFileSync(envPath, newContent) |
|||
}) |
|||
} catch (err) { |
|||
/* istanbul ignore next */ |
|||
ctx.throw(400, err) |
|||
} |
|||
} |
|||
|
|||
@ -1,28 +1,10 @@ |
|||
const { performDump } = require("../../utilities/templates") |
|||
const path = require("path") |
|||
const os = require("os") |
|||
const fs = require("fs-extra") |
|||
const { performBackup } = require("../../utilities/fileSystem") |
|||
|
|||
exports.exportAppDump = async function(ctx) { |
|||
const { appId } = ctx.query |
|||
|
|||
const appname = decodeURI(ctx.query.appname) |
|||
|
|||
const backupsDir = path.join(os.homedir(), ".budibase", "backups") |
|||
fs.ensureDirSync(backupsDir) |
|||
|
|||
const backupIdentifier = `${appname}Backup${new Date().getTime()}.txt` |
|||
|
|||
await performDump({ |
|||
dir: backupsDir, |
|||
appId, |
|||
name: backupIdentifier, |
|||
}) |
|||
|
|||
ctx.status = 200 |
|||
|
|||
const backupFile = path.join(backupsDir, backupIdentifier) |
|||
|
|||
ctx.attachment(backupIdentifier) |
|||
ctx.body = fs.createReadStream(backupFile) |
|||
ctx.body = await performBackup(appId, backupIdentifier) |
|||
} |
|||
|
|||
@ -1,44 +1,30 @@ |
|||
const CouchDB = require("../../db") |
|||
const { resolve, join } = require("../../utilities/centralPath") |
|||
const { |
|||
budibaseTempDir, |
|||
budibaseAppsDir, |
|||
} = require("../../utilities/budibaseDir") |
|||
const { getComponentLibraryManifest } = require("../../utilities/fileSystem") |
|||
|
|||
exports.fetchAppComponentDefinitions = async function(ctx) { |
|||
const appId = ctx.params.appId || ctx.appId |
|||
const db = new CouchDB(appId) |
|||
const app = await db.get(appId) |
|||
|
|||
ctx.body = app.componentLibraries.reduce((acc, componentLibrary) => { |
|||
let appDirectory = resolve(budibaseAppsDir(), appId, "node_modules") |
|||
let componentManifests = await Promise.all( |
|||
app.componentLibraries.map(async library => { |
|||
let manifest = await getComponentLibraryManifest(appId, library) |
|||
|
|||
if (ctx.isDev) { |
|||
appDirectory = budibaseTempDir() |
|||
} |
|||
|
|||
const componentJson = require(join( |
|||
appDirectory, |
|||
componentLibrary, |
|||
ctx.isDev ? "" : "package", |
|||
"manifest.json" |
|||
)) |
|||
|
|||
const result = {} |
|||
|
|||
// map over the components.json and add the library identifier as a key
|
|||
// button -> @budibase/standard-components/button
|
|||
for (let key of Object.keys(componentJson)) { |
|||
const fullComponentName = `${componentLibrary}/${key}`.toLowerCase() |
|||
result[fullComponentName] = { |
|||
return { |
|||
manifest, |
|||
library, |
|||
} |
|||
}) |
|||
) |
|||
const definitions = {} |
|||
for (let { manifest, library } of componentManifests) { |
|||
for (let key of Object.keys(manifest)) { |
|||
const fullComponentName = `${library}/${key}`.toLowerCase() |
|||
definitions[fullComponentName] = { |
|||
component: fullComponentName, |
|||
...componentJson[key], |
|||
...manifest[key], |
|||
} |
|||
} |
|||
|
|||
return { |
|||
...acc, |
|||
...result, |
|||
} |
|||
}, {}) |
|||
} |
|||
ctx.body = definitions |
|||
} |
|||
|
|||
@ -1,173 +0,0 @@ |
|||
<!DOCTYPE html> |
|||
<html lang="en"> |
|||
<head> |
|||
<meta charset="UTF-8"> |
|||
<title>Budibase self hosting️</title> |
|||
<style> |
|||
body { |
|||
font-family: Inter, -apple-system, BlinkMacSystemFont, Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol"; |
|||
height: 100%; |
|||
width: 100%; |
|||
margin: 0; |
|||
padding: 0; |
|||
background: #fafafa; |
|||
} |
|||
|
|||
.main { |
|||
padding: 0 20px; |
|||
margin: 30px auto; |
|||
width: 60%; |
|||
} |
|||
|
|||
h2 { |
|||
font-size: clamp(24px, 1.5vw, 30px); |
|||
text-align: center; |
|||
line-height: 1.3; |
|||
font-weight: bold; |
|||
} |
|||
|
|||
.card-grid { |
|||
display: grid; |
|||
grid-template-columns: 1fr 1fr; |
|||
gap: 3rem; |
|||
} |
|||
|
|||
.card { |
|||
display: grid; |
|||
background-color: #222222; |
|||
grid-template-columns: 1fr; |
|||
align-items: center; |
|||
padding: 2.5rem 1.75rem; |
|||
border-radius: 12px; |
|||
color: white; |
|||
} |
|||
|
|||
.card h3 { |
|||
margin: 0; |
|||
font-size: 24px; |
|||
font-family: sans-serif; |
|||
font-weight: 600; |
|||
} |
|||
|
|||
.card h3 b { |
|||
text-wrap: normal; |
|||
font-size: 36px; |
|||
padding-right: 14px; |
|||
} |
|||
|
|||
.card p { |
|||
color: #ffffff; |
|||
opacity: 0.8; |
|||
font-size: 18px; |
|||
text-align: left; |
|||
line-height: 1.3; |
|||
margin-top: 1rem; |
|||
} |
|||
|
|||
.logo { |
|||
width: 60px; |
|||
height: 60px; |
|||
margin: auto; |
|||
} |
|||
|
|||
.top-text { |
|||
text-align: center; |
|||
color: #707070; |
|||
margin: 0 0 1.5rem 0; |
|||
} |
|||
|
|||
.button { |
|||
cursor: pointer; |
|||
display: block; |
|||
background: #4285f4; |
|||
color: white; |
|||
padding: 12px 16px; |
|||
font-size: 16px; |
|||
font-weight: 600; |
|||
border-radius: 6px; |
|||
max-width: 120px; |
|||
text-align: center; |
|||
transition: 200ms background ease; |
|||
text-decoration: none; |
|||
} |
|||
|
|||
.info { |
|||
background: #f5f5f5; |
|||
padding: 1rem 1rem 1rem 1rem; |
|||
border: #ccc 1px solid; |
|||
border-radius: 6px; |
|||
margin-top: 40px; |
|||
display: flex; |
|||
align-items: center; |
|||
} |
|||
|
|||
.info p { |
|||
margin-left: 20px; |
|||
color: #222222; |
|||
font-family: sans-serif; |
|||
} |
|||
|
|||
.info p { |
|||
margin-right: 20px; |
|||
} |
|||
|
|||
.info svg { |
|||
margin-left: 20px; |
|||
} |
|||
|
|||
.info a { |
|||
color: #4285f4; |
|||
} |
|||
</style> |
|||
</head> |
|||
<body> |
|||
<div class="main"> |
|||
<div class="logo"> |
|||
{{logo}} |
|||
</div> |
|||
<h2>Get started with Budibase Self Hosting</h2> |
|||
<p class="top-text">Use the address <b id="url"></b> in your Builder</p> |
|||
<div class="card-grid"> |
|||
<div class="card"> |
|||
<h3><b>📚</b>Documentation</h3> |
|||
<p> |
|||
Find out more about your self hosted platform. |
|||
</p> |
|||
<a class="button" |
|||
href="https://docs.budibase.com/self-hosting/introduction-to-self-hosting"> |
|||
Documentation |
|||
</a> |
|||
</div> |
|||
<div class="card"> |
|||
<h3><b>💻</b>Next steps</h3> |
|||
<p> |
|||
Find out how to make use of your self hosted Budibase platform. |
|||
</p> |
|||
<a class="button" |
|||
href="https://docs.budibase.com/self-hosting/builder-settings"> |
|||
Next steps |
|||
</a> |
|||
</div> |
|||
</div> |
|||
<div class="info"> |
|||
<svg preserveAspectRatio="xMidYMid meet" height="28px" width="28px" fill="currentColor" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" |
|||
xmlns:xlink="http://www.w3.org/1999/xlink" stroke="none" class="icon-7f6730be--text-3f89f380"> |
|||
<g> |
|||
<path d="M12.2 8.98c.06-.01.12-.03.18-.06.06-.02.12-.05.18-.09l.15-.12c.18-.19.29-.45.29-.71 0-.06-.01-.13-.02-.19a.603.603 0 0 0-.06-.19.757.757 0 0 0-.09-.18c-.03-.05-.08-.1-.12-.15-.28-.27-.72-.37-1.09-.21-.13.05-.23.12-.33.21-.04.05-.09.1-.12.15-.04.06-.07.12-.09.18-.03.06-.05.12-.06.19-.01.06-.02.13-.02.19 0 .26.11.52.29.71.1.09.2.16.33.21.12.05.25.08.38.08.06 0 .13-.01.2-.02M13 16v-4a1 1 0 1 0-2 0v4a1 1 0 1 0 2 0M12 3c-4.962 0-9 4.038-9 9 0 4.963 4.038 9 9 9 4.963 0 9-4.037 9-9 0-4.962-4.037-9-9-9m0 20C5.935 23 1 18.065 1 12S5.935 1 12 1c6.066 0 11 4.935 11 11s-4.934 11-11 11" fill-rule="evenodd"> |
|||
</path> |
|||
</g> |
|||
</svg> |
|||
<p>A <b>Hosting Key</b> will also be required, this can be found in your hosting properties, info found <a href="https://docs.budibase.com/self-hosting/hosting-settings">here</a>.</p> |
|||
</div> |
|||
</div> |
|||
<script> |
|||
window.addEventListener("load", () => { |
|||
let url = document.URL.split("//")[1] |
|||
if (url.substring(url.length - 1) === "/") { |
|||
url = url.substring(0, url.length - 1) |
|||
} |
|||
document.getElementById("url").innerHTML = url |
|||
}) |
|||
</script> |
|||
</body> |
|||
</html> |
|||
|
Before Width: | Height: | Size: 4.6 KiB |
@ -1,16 +0,0 @@ |
|||
const setup = require("./utilities") |
|||
|
|||
describe("test things in the Cloud/Self hosted", () => { |
|||
describe("test self hosted static page", () => { |
|||
it("should be able to load the static page", async () => { |
|||
await setup.switchToCloudForFunction(async () => { |
|||
let request = setup.getRequest() |
|||
let config = setup.getConfig() |
|||
await config.init() |
|||
const res = await request.get(`/`).expect(200) |
|||
expect(res.text.includes("<title>Budibase self hosting️</title>")).toEqual(true) |
|||
setup.afterAll() |
|||
}) |
|||
}) |
|||
}) |
|||
}) |
|||
@ -0,0 +1,38 @@ |
|||
const CouchDB = require("./index") |
|||
const { StaticDatabases } = require("./utils") |
|||
const env = require("../environment") |
|||
|
|||
const SELF_HOST_ERR = "Unable to access builder DB/doc - not self hosted." |
|||
const BUILDER_DB = StaticDatabases.BUILDER |
|||
|
|||
/** |
|||
* This is the builder database, right now this is a single, static database |
|||
* that is present across the whole system and determines some core functionality |
|||
* for the builder (e.g. storage of API keys). This has been limited to self hosting |
|||
* as it doesn't make as much sense against the currently design Cloud system. |
|||
*/ |
|||
|
|||
exports.getBuilderMainDoc = async () => { |
|||
if (!env.SELF_HOSTED) { |
|||
throw SELF_HOST_ERR |
|||
} |
|||
const db = new CouchDB(BUILDER_DB.name) |
|||
try { |
|||
return await db.get(BUILDER_DB.baseDoc) |
|||
} catch (err) { |
|||
// doesn't exist yet, nothing to get
|
|||
return { |
|||
_id: BUILDER_DB.baseDoc, |
|||
} |
|||
} |
|||
} |
|||
|
|||
exports.setBuilderMainDoc = async doc => { |
|||
if (!env.SELF_HOSTED) { |
|||
throw SELF_HOST_ERR |
|||
} |
|||
// make sure to override the ID
|
|||
doc._id = BUILDER_DB.baseDoc |
|||
const db = new CouchDB(BUILDER_DB.name) |
|||
return db.put(doc) |
|||
} |
|||
@ -1,112 +1,112 @@ |
|||
const { app, BrowserWindow, shell, dialog } = require("electron") |
|||
const { join } = require("./utilities/centralPath") |
|||
const isDev = require("electron-is-dev") |
|||
const { autoUpdater } = require("electron-updater") |
|||
const unhandled = require("electron-unhandled") |
|||
const { existsSync } = require("fs-extra") |
|||
const initialiseBudibase = require("./utilities/initialiseBudibase") |
|||
const { budibaseAppsDir } = require("./utilities/budibaseDir") |
|||
const { openNewGitHubIssue, debugInfo } = require("electron-util") |
|||
const eventEmitter = require("./events") |
|||
// const { app, BrowserWindow, shell, dialog } = require("electron")
|
|||
// const { join } = require("./utilities/centralPath")
|
|||
// const isDev = require("electron-is-dev")
|
|||
// const { autoUpdater } = require("electron-updater")
|
|||
// const unhandled = require("electron-unhandled")
|
|||
// const { existsSync } = require("fs-extra")
|
|||
// const initialiseBudibase = require("./utilities/initialiseBudibase")
|
|||
// const { budibaseAppsDir } = require("./utilities/budibaseDir")
|
|||
// const { openNewGitHubIssue, debugInfo } = require("electron-util")
|
|||
// const eventEmitter = require("./events")
|
|||
|
|||
const budibaseDir = budibaseAppsDir() |
|||
const envFile = join(budibaseDir, ".env") |
|||
// const budibaseDir = budibaseAppsDir()
|
|||
// const envFile = join(budibaseDir, ".env")
|
|||
|
|||
async function startApp() { |
|||
if (!existsSync(envFile)) { |
|||
await initialiseBudibase({ dir: budibaseDir }) |
|||
} |
|||
// evict environment from cache, so it reloads when next asked
|
|||
delete require.cache[require.resolve("./environment")] |
|||
// store the port incase its going to get overridden
|
|||
const port = process.env.PORT |
|||
require("dotenv").config({ path: envFile }) |
|||
// overwrite the port - don't want to use dotenv for the port
|
|||
require("./environment")._set("PORT", port) |
|||
// async function startApp() {
|
|||
// if (!existsSync(envFile)) {
|
|||
// await initialiseBudibase({ dir: budibaseDir })
|
|||
// }
|
|||
// // evict environment from cache, so it reloads when next asked
|
|||
// delete require.cache[require.resolve("./environment")]
|
|||
// // store the port incase its going to get overridden
|
|||
// const port = process.env.PORT
|
|||
// require("dotenv").config({ path: envFile })
|
|||
// // overwrite the port - don't want to use dotenv for the port
|
|||
// require("./environment")._set("PORT", port)
|
|||
|
|||
unhandled({ |
|||
showDialog: true, |
|||
reportButton: error => { |
|||
openNewGitHubIssue({ |
|||
title: error.message, |
|||
user: "Budibase", |
|||
labels: ["error-report"], |
|||
repo: "budibase", |
|||
body: `### Error that occurred when using the budibase builder:\n\`\`\`\n${ |
|||
error.stack |
|||
}\n\`\`\`\n### Operating System Information:\n---\n\n${debugInfo()}`, |
|||
}) |
|||
}, |
|||
}) |
|||
// unhandled({
|
|||
// showDialog: true,
|
|||
// reportButton: error => {
|
|||
// openNewGitHubIssue({
|
|||
// title: error.message,
|
|||
// user: "Budibase",
|
|||
// labels: ["error-report"],
|
|||
// repo: "budibase",
|
|||
// body: `### Error that occurred when using the budibase builder:\n\`\`\`\n${
|
|||
// error.stack
|
|||
// }\n\`\`\`\n### Operating System Information:\n---\n\n${debugInfo()}`,
|
|||
// })
|
|||
// },
|
|||
// })
|
|||
|
|||
let win |
|||
// let win
|
|||
|
|||
function handleRedirect(e, url) { |
|||
e.preventDefault() |
|||
shell.openExternal(url) |
|||
} |
|||
// function handleRedirect(e, url) {
|
|||
// e.preventDefault()
|
|||
// shell.openExternal(url)
|
|||
// }
|
|||
|
|||
async function createWindow() { |
|||
app.server = require("./app") |
|||
eventEmitter.on("internal:port", port => { |
|||
const APP_URL = `http://localhost:${port}/_builder` |
|||
const APP_TITLE = "Budibase Builder" |
|||
win = new BrowserWindow({ |
|||
width: 1920, |
|||
height: 1080, |
|||
icon: join(__dirname, "..", "build", "icons", "512x512.png"), |
|||
}) |
|||
win.setTitle(APP_TITLE) |
|||
win.loadURL(APP_URL) |
|||
if (isDev) { |
|||
win.webContents.openDevTools() |
|||
} else { |
|||
autoUpdater.checkForUpdatesAndNotify() |
|||
} |
|||
// async function createWindow() {
|
|||
// app.server = require("./app")
|
|||
// eventEmitter.on("internal:port", port => {
|
|||
// const APP_URL = `http://localhost:${port}/_builder`
|
|||
// const APP_TITLE = "Budibase Builder"
|
|||
// win = new BrowserWindow({
|
|||
// width: 1920,
|
|||
// height: 1080,
|
|||
// icon: join(__dirname, "..", "build", "icons", "512x512.png"),
|
|||
// })
|
|||
// win.setTitle(APP_TITLE)
|
|||
// win.loadURL(APP_URL)
|
|||
// if (isDev) {
|
|||
// win.webContents.openDevTools()
|
|||
// } else {
|
|||
// autoUpdater.checkForUpdatesAndNotify()
|
|||
// }
|
|||
|
|||
// open _blank in default browser
|
|||
win.webContents.on("new-window", handleRedirect) |
|||
win.webContents.on("will-navigate", handleRedirect) |
|||
}) |
|||
} |
|||
// // open _blank in default browser
|
|||
// win.webContents.on("new-window", handleRedirect)
|
|||
// win.webContents.on("will-navigate", handleRedirect)
|
|||
// })
|
|||
// }
|
|||
|
|||
app.whenReady().then(createWindow) |
|||
// app.whenReady().then(createWindow)
|
|||
|
|||
// Quit when all windows are closed.
|
|||
app.on("window-all-closed", () => { |
|||
// On macOS it is common for applications and their menu bar
|
|||
// to stay active until the user quits explicitly with Cmd + Q
|
|||
if (process.platform !== "darwin") { |
|||
app.server.close() |
|||
app.quit() |
|||
} |
|||
}) |
|||
// // Quit when all windows are closed.
|
|||
// app.on("window-all-closed", () => {
|
|||
// // On macOS it is common for applications and their menu bar
|
|||
// // to stay active until the user quits explicitly with Cmd + Q
|
|||
// if (process.platform !== "darwin") {
|
|||
// app.server.close()
|
|||
// app.quit()
|
|||
// }
|
|||
// })
|
|||
|
|||
app.on("activate", () => { |
|||
// On macOS it's common to re-create a window in the app when the
|
|||
// dock icon is clicked and there are no other windows open.
|
|||
if (win === null) createWindow() |
|||
}) |
|||
} |
|||
// app.on("activate", () => {
|
|||
// // On macOS it's common to re-create a window in the app when the
|
|||
// // dock icon is clicked and there are no other windows open.
|
|||
// if (win === null) createWindow()
|
|||
// })
|
|||
// }
|
|||
|
|||
autoUpdater.on("update-downloaded", (event, releaseNotes, releaseName) => { |
|||
const dialogOpts = { |
|||
type: "info", |
|||
buttons: ["Restart", "Later"], |
|||
title: "Budibase Update Available", |
|||
message: process.platform === "win32" ? releaseNotes : releaseName, |
|||
detail: |
|||
"A new version of the budibase builder has been downloaded. Restart the application to apply the updates.", |
|||
} |
|||
// autoUpdater.on("update-downloaded", (event, releaseNotes, releaseName) => {
|
|||
// const dialogOpts = {
|
|||
// type: "info",
|
|||
// buttons: ["Restart", "Later"],
|
|||
// title: "Budibase Update Available",
|
|||
// message: process.platform === "win32" ? releaseNotes : releaseName,
|
|||
// detail:
|
|||
// "A new version of the budibase builder has been downloaded. Restart the application to apply the updates.",
|
|||
// }
|
|||
|
|||
dialog.showMessageBox(dialogOpts).then(returnValue => { |
|||
if (returnValue.response === 0) autoUpdater.quitAndInstall() |
|||
}) |
|||
}) |
|||
// dialog.showMessageBox(dialogOpts).then(returnValue => {
|
|||
// if (returnValue.response === 0) autoUpdater.quitAndInstall()
|
|||
// })
|
|||
// })
|
|||
|
|||
autoUpdater.on("error", message => { |
|||
console.error("There was a problem updating the application") |
|||
console.error(message) |
|||
}) |
|||
// autoUpdater.on("error", message => {
|
|||
// console.error("There was a problem updating the application")
|
|||
// console.error(message)
|
|||
// })
|
|||
|
|||
startApp() |
|||
// startApp()
|
|||
|
|||
@ -1,45 +1,64 @@ |
|||
const { resolve, join } = require("./utilities/centralPath") |
|||
const { homedir } = require("os") |
|||
const { app } = require("electron") |
|||
function isTest() { |
|||
return ( |
|||
process.env.NODE_ENV === "jest" || |
|||
process.env.NODE_ENV === "cypress" || |
|||
process.env.JEST_WORKER_ID != null |
|||
) |
|||
} |
|||
|
|||
let LOADED = false |
|||
function isDev() { |
|||
return ( |
|||
process.env.NODE_ENV !== "production" && |
|||
process.env.BUDIBASE_ENVIRONMENT !== "production" |
|||
) |
|||
} |
|||
|
|||
if (!LOADED) { |
|||
const homeDir = app ? app.getPath("home") : homedir() |
|||
const budibaseDir = join(homeDir, ".budibase") |
|||
process.env.BUDIBASE_DIR = budibaseDir |
|||
require("dotenv").config({ path: resolve(budibaseDir, ".env") }) |
|||
let LOADED = false |
|||
if (!LOADED && isDev() && !isTest()) { |
|||
require("dotenv").config() |
|||
LOADED = true |
|||
} |
|||
|
|||
module.exports = { |
|||
CLIENT_ID: process.env.CLIENT_ID, |
|||
NODE_ENV: process.env.NODE_ENV, |
|||
JWT_SECRET: process.env.JWT_SECRET, |
|||
BUDIBASE_DIR: process.env.BUDIBASE_DIR, |
|||
// important
|
|||
PORT: process.env.PORT, |
|||
JWT_SECRET: process.env.JWT_SECRET, |
|||
COUCH_DB_URL: process.env.COUCH_DB_URL, |
|||
MINIO_URL: process.env.MINIO_URL, |
|||
WORKER_URL: process.env.WORKER_URL, |
|||
SELF_HOSTED: process.env.SELF_HOSTED, |
|||
AWS_REGION: process.env.AWS_REGION, |
|||
ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS, |
|||
MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY, |
|||
MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY, |
|||
USE_QUOTAS: process.env.USE_QUOTAS, |
|||
// environment
|
|||
NODE_ENV: process.env.NODE_ENV, |
|||
JEST_WORKER_ID: process.env.JEST_WORKER_ID, |
|||
BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT, |
|||
// minor
|
|||
SALT_ROUNDS: process.env.SALT_ROUNDS, |
|||
LOGGER: process.env.LOGGER, |
|||
LOG_LEVEL: process.env.LOG_LEVEL, |
|||
AUTOMATION_DIRECTORY: process.env.AUTOMATION_DIRECTORY, |
|||
AUTOMATION_BUCKET: process.env.AUTOMATION_BUCKET, |
|||
BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT, |
|||
SENDGRID_API_KEY: process.env.SENDGRID_API_KEY, |
|||
CLOUD: process.env.CLOUD, |
|||
SELF_HOSTED: process.env.SELF_HOSTED, |
|||
WORKER_URL: process.env.WORKER_URL, |
|||
HOSTING_KEY: process.env.HOSTING_KEY, |
|||
DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT, |
|||
AWS_REGION: process.env.AWS_REGION, |
|||
DEPLOYMENT_CREDENTIALS_URL: process.env.DEPLOYMENT_CREDENTIALS_URL, |
|||
// old - to remove
|
|||
CLIENT_ID: process.env.CLIENT_ID, |
|||
BUDIBASE_DIR: process.env.BUDIBASE_DIR, |
|||
DEPLOYMENT_DB_URL: process.env.DEPLOYMENT_DB_URL, |
|||
BUDIBASE_API_KEY: process.env.BUDIBASE_API_KEY, |
|||
USERID_API_KEY: process.env.USERID_API_KEY, |
|||
ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS, |
|||
DEPLOYMENT_DB_URL: process.env.DEPLOYMENT_DB_URL, |
|||
LOCAL_TEMPLATES: process.env.LOCAL_TEMPLATES, |
|||
DEPLOYMENT_CREDENTIALS_URL: process.env.DEPLOYMENT_CREDENTIALS_URL, |
|||
HOSTING_KEY: process.env.HOSTING_KEY, |
|||
_set(key, value) { |
|||
process.env[key] = value |
|||
module.exports[key] = value |
|||
}, |
|||
isTest, |
|||
isDev, |
|||
isProd: () => { |
|||
return !isDev() |
|||
}, |
|||
} |
|||
|
|||
@ -1,35 +0,0 @@ |
|||
const { ensureDir, constants, copyFile } = require("fs-extra") |
|||
const { join } = require("../centralPath") |
|||
const { budibaseAppsDir } = require("../budibaseDir") |
|||
|
|||
/** |
|||
* Compile all the non-db static web assets that are required for the running of |
|||
* a budibase application. This includes the JSON structure of the DOM and |
|||
* the client library, a script responsible for reading the JSON structure |
|||
* and rendering the application. |
|||
* @param {string} appId id of the application we want to compile static assets for |
|||
*/ |
|||
module.exports = async appId => { |
|||
const publicPath = join(budibaseAppsDir(), appId, "public") |
|||
await ensureDir(publicPath) |
|||
await copyClientLib(publicPath) |
|||
} |
|||
|
|||
/** |
|||
* Copy the budibase client library and sourcemap from NPM to <appId>/public/. |
|||
* The client library is then served as a static asset when the budibase application |
|||
* is running in preview or prod |
|||
* @param {String} publicPath - path to write the client library to |
|||
*/ |
|||
const copyClientLib = async publicPath => { |
|||
const sourcepath = require.resolve("@budibase/client") |
|||
const destPath = join(publicPath, "budibase-client.js") |
|||
|
|||
await copyFile(sourcepath, destPath, constants.COPYFILE_FICLONE) |
|||
|
|||
await copyFile( |
|||
sourcepath + ".map", |
|||
destPath + ".map", |
|||
constants.COPYFILE_FICLONE |
|||
) |
|||
} |
|||
@ -1,29 +0,0 @@ |
|||
const stream = require("stream") |
|||
const fetch = require("node-fetch") |
|||
const tar = require("tar-fs") |
|||
const zlib = require("zlib") |
|||
const { promisify } = require("util") |
|||
const packageJson = require("../../package.json") |
|||
|
|||
const streamPipeline = promisify(stream.pipeline) |
|||
|
|||
// can't really test this due to the downloading nature of it, wouldn't be a great test case
|
|||
/* istanbul ignore next */ |
|||
exports.downloadExtractComponentLibraries = async appFolder => { |
|||
const LIBRARIES = ["standard-components"] |
|||
|
|||
// Need to download tarballs directly from NPM as our users may not have node on their machine
|
|||
for (let lib of LIBRARIES) { |
|||
// download tarball
|
|||
const registryUrl = `https://registry.npmjs.org/@budibase/${lib}/-/${lib}-${packageJson.version}.tgz` |
|||
const response = await fetch(registryUrl) |
|||
if (!response.ok) |
|||
throw new Error(`unexpected response ${response.statusText}`) |
|||
|
|||
await streamPipeline( |
|||
response.body, |
|||
zlib.Unzip(), |
|||
tar.extract(`${appFolder}/node_modules/@budibase/${lib}`) |
|||
) |
|||
} |
|||
} |
|||
@ -0,0 +1,232 @@ |
|||
const { budibaseTempDir } = require("../budibaseDir") |
|||
const { isDev } = require("../index") |
|||
const fs = require("fs") |
|||
const { join } = require("path") |
|||
const uuid = require("uuid/v4") |
|||
const CouchDB = require("../../db") |
|||
const { ObjectStoreBuckets } = require("../../constants") |
|||
const { |
|||
upload, |
|||
retrieve, |
|||
retrieveToTmp, |
|||
streamUpload, |
|||
deleteFolder, |
|||
downloadTarball, |
|||
} = require("./utilities") |
|||
const { downloadLibraries, newAppPublicPath } = require("./newApp") |
|||
const download = require("download") |
|||
const env = require("../../environment") |
|||
const { homedir } = require("os") |
|||
const fetch = require("node-fetch") |
|||
|
|||
const DEFAULT_AUTOMATION_BUCKET = |
|||
"https://prod-budi-automations.s3-eu-west-1.amazonaws.com" |
|||
const DEFAULT_AUTOMATION_DIRECTORY = ".budibase-automations" |
|||
|
|||
/** |
|||
* The single stack system (Cloud and Builder) should not make use of the file system where possible, |
|||
* this file handles all of the file access for the system with the intention of limiting it all to one |
|||
* place. Keeping all of this logic in one place means that when we need to do file system access (like |
|||
* downloading a package or opening a temporary file) in can be done in way that we can confirm it shouldn't |
|||
* be done through an object store instead. |
|||
*/ |
|||
|
|||
/** |
|||
* Upon first startup of instance there may not be everything we need in tmp directory, set it up. |
|||
*/ |
|||
exports.init = () => { |
|||
const tempDir = budibaseTempDir() |
|||
if (!fs.existsSync(tempDir)) { |
|||
fs.mkdirSync(tempDir) |
|||
} |
|||
const clientLibPath = join(budibaseTempDir(), "budibase-client.js") |
|||
if (env.isTest() && !fs.existsSync(clientLibPath)) { |
|||
fs.copyFileSync(require.resolve("@budibase/client"), clientLibPath) |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* Checks if the system is currently in development mode and if it is makes sure |
|||
* everything required to function is ready. |
|||
*/ |
|||
exports.checkDevelopmentEnvironment = () => { |
|||
if (!isDev()) { |
|||
return |
|||
} |
|||
let error |
|||
if (!fs.existsSync(budibaseTempDir())) { |
|||
error = |
|||
"Please run a build before attempting to run server independently to fill 'tmp' directory." |
|||
} |
|||
if (!fs.existsSync(join(process.cwd(), ".env"))) { |
|||
error = "Must run via yarn once to generate environment." |
|||
} |
|||
if (error) { |
|||
console.error(error) |
|||
process.exit(-1) |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* This function manages temporary template files which are stored by Koa. |
|||
* @param {Object} template The template object retrieved from the Koa context object. |
|||
* @returns {Object} Returns an fs read stream which can be loaded into the database. |
|||
*/ |
|||
exports.getTemplateStream = async template => { |
|||
if (template.file) { |
|||
return fs.createReadStream(template.file.path) |
|||
} else { |
|||
const tmpPath = await exports.downloadTemplate(...template.key.split("/")) |
|||
return fs.createReadStream(join(tmpPath, "db", "dump.txt")) |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* Used to retrieve a handlebars file from the system which will be used as a template. |
|||
* This is allowable as the template handlebars files should be static and identical across |
|||
* the cluster. |
|||
* @param {string} path The path to the handlebars file which is to be loaded. |
|||
* @returns {string} The loaded handlebars file as a string - loaded as utf8. |
|||
*/ |
|||
exports.loadHandlebarsFile = path => { |
|||
return fs.readFileSync(path, "utf8") |
|||
} |
|||
|
|||
/** |
|||
* When return a file from the API need to write the file to the system temporarily so we |
|||
* can create a read stream to send. |
|||
* @param {string} contents the contents of the file which is to be returned from the API. |
|||
* @return {Object} the read stream which can be put into the koa context body. |
|||
*/ |
|||
exports.apiFileReturn = contents => { |
|||
const path = join(budibaseTempDir(), uuid()) |
|||
fs.writeFileSync(path, contents) |
|||
return fs.createReadStream(path) |
|||
} |
|||
|
|||
/** |
|||
* Takes a copy of the database state for an app to the object store. |
|||
* @param {string} appId The ID of the app which is to be backed up. |
|||
* @param {string} backupName The name of the backup located in the object store. |
|||
* @return The backup has been completed when this promise completes and returns a file stream |
|||
* to the temporary backup file (to return via API if required). |
|||
*/ |
|||
exports.performBackup = async (appId, backupName) => { |
|||
const path = join(budibaseTempDir(), backupName) |
|||
const writeStream = fs.createWriteStream(path) |
|||
// perform couch dump
|
|||
const instanceDb = new CouchDB(appId) |
|||
await instanceDb.dump(writeStream, {}) |
|||
// write the file to the object store
|
|||
await streamUpload( |
|||
ObjectStoreBuckets.BACKUPS, |
|||
join(appId, backupName), |
|||
fs.createReadStream(path) |
|||
) |
|||
return fs.createReadStream(path) |
|||
} |
|||
|
|||
/** |
|||
* Downloads required libraries and creates a new path in the object store. |
|||
* @param {string} appId The ID of the app which is being created. |
|||
* @return {Promise<void>} once promise completes app resources should be ready in object store. |
|||
*/ |
|||
exports.createApp = async appId => { |
|||
await downloadLibraries(appId) |
|||
await newAppPublicPath(appId) |
|||
} |
|||
|
|||
/** |
|||
* Removes all of the assets created for an app in the object store. |
|||
* @param {string} appId The ID of the app which is being deleted. |
|||
* @return {Promise<void>} once promise completes the app resources will be removed from object store. |
|||
*/ |
|||
exports.deleteApp = async appId => { |
|||
await deleteFolder(ObjectStoreBuckets.APPS, `${appId}/`) |
|||
} |
|||
|
|||
/** |
|||
* Retrieves a template and pipes it to minio as well as making it available temporarily. |
|||
* @param {string} type The type of template which is to be retrieved. |
|||
* @param name |
|||
* @return {Promise<*>} |
|||
*/ |
|||
exports.downloadTemplate = async (type, name) => { |
|||
const DEFAULT_TEMPLATES_BUCKET = |
|||
"prod-budi-templates.s3-eu-west-1.amazonaws.com" |
|||
const templateUrl = `https://${DEFAULT_TEMPLATES_BUCKET}/templates/${type}/${name}.tar.gz` |
|||
return downloadTarball(templateUrl, ObjectStoreBuckets.TEMPLATES, type) |
|||
} |
|||
|
|||
/** |
|||
* Retrieves component libraries from object store (or tmp symlink if in local) |
|||
*/ |
|||
exports.getComponentLibraryManifest = async (appId, library) => { |
|||
const filename = "manifest.json" |
|||
/* istanbul ignore next */ |
|||
// when testing in cypress and so on we need to get the package
|
|||
// as the environment may not be fully fleshed out for dev or prod
|
|||
if (env.isTest()) { |
|||
const lib = library.split("/")[1] |
|||
const path = require.resolve(library).split(lib)[0] |
|||
return require(join(path, lib, filename)) |
|||
} |
|||
const devPath = join(budibaseTempDir(), library, filename) |
|||
if (env.isDev() && fs.existsSync(devPath)) { |
|||
return require(devPath) |
|||
} |
|||
const path = join(appId, "node_modules", library, "package", filename) |
|||
let resp = await retrieve(ObjectStoreBuckets.APPS, path) |
|||
if (typeof resp !== "string") { |
|||
resp = resp.toString("utf8") |
|||
} |
|||
return JSON.parse(resp) |
|||
} |
|||
|
|||
exports.automationInit = async () => { |
|||
const directory = |
|||
env.AUTOMATION_DIRECTORY || join(homedir(), DEFAULT_AUTOMATION_DIRECTORY) |
|||
const bucket = env.AUTOMATION_BUCKET || DEFAULT_AUTOMATION_BUCKET |
|||
if (!fs.existsSync(directory)) { |
|||
fs.mkdirSync(directory, { recursive: true }) |
|||
} |
|||
// env setup to get async packages
|
|||
let response = await fetch(`${bucket}/manifest.json`) |
|||
return response.json() |
|||
} |
|||
|
|||
exports.getExternalAutomationStep = async (name, version, bundleName) => { |
|||
const directory = |
|||
env.AUTOMATION_DIRECTORY || join(homedir(), DEFAULT_AUTOMATION_DIRECTORY) |
|||
const bucket = env.AUTOMATION_BUCKET || DEFAULT_AUTOMATION_BUCKET |
|||
try { |
|||
return require(join(directory, bundleName)) |
|||
} catch (err) { |
|||
await download(`${bucket}/${name}/${version}/${bundleName}`, directory) |
|||
return require(join(directory, bundleName)) |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* All file reads come through here just to make sure all of them make sense |
|||
* allows a centralised location to check logic is all good. |
|||
*/ |
|||
exports.readFileSync = (filepath, options = "utf8") => { |
|||
return fs.readFileSync(filepath, options) |
|||
} |
|||
|
|||
/** |
|||
* Given a set of app IDs makes sure file system is cleared of any of their temp info. |
|||
*/ |
|||
exports.cleanup = appIds => { |
|||
for (let appId of appIds) { |
|||
fs.rmdirSync(join(budibaseTempDir(), appId), { recursive: true }) |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* Full function definition for below can be found in the utilities. |
|||
*/ |
|||
exports.upload = upload |
|||
exports.retrieve = retrieve |
|||
exports.retrieveToTmp = retrieveToTmp |
|||
@ -0,0 +1,40 @@ |
|||
const packageJson = require("../../../package.json") |
|||
const { join } = require("path") |
|||
const { ObjectStoreBuckets } = require("../../constants") |
|||
const { streamUpload, downloadTarball } = require("./utilities") |
|||
const fs = require("fs") |
|||
|
|||
const BUCKET_NAME = ObjectStoreBuckets.APPS |
|||
|
|||
// can't really test this due to the downloading nature of it, wouldn't be a great test case
|
|||
/* istanbul ignore next */ |
|||
exports.downloadLibraries = async appId => { |
|||
const LIBRARIES = ["standard-components"] |
|||
|
|||
const paths = {} |
|||
// Need to download tarballs directly from NPM as our users may not have node on their machine
|
|||
for (let lib of LIBRARIES) { |
|||
// download tarball
|
|||
const registryUrl = `https://registry.npmjs.org/@budibase/${lib}/-/${lib}-${packageJson.version}.tgz` |
|||
const path = join(appId, "node_modules", "@budibase", lib) |
|||
paths[`@budibase/${lib}`] = await downloadTarball( |
|||
registryUrl, |
|||
BUCKET_NAME, |
|||
path |
|||
) |
|||
} |
|||
return paths |
|||
} |
|||
|
|||
exports.newAppPublicPath = async appId => { |
|||
const path = join(appId, "public") |
|||
const sourcepath = require.resolve("@budibase/client") |
|||
const destPath = join(path, "budibase-client.js") |
|||
|
|||
await streamUpload(BUCKET_NAME, destPath, fs.createReadStream(sourcepath)) |
|||
await streamUpload( |
|||
BUCKET_NAME, |
|||
destPath + ".map", |
|||
fs.createReadStream(sourcepath + ".map") |
|||
) |
|||
} |
|||
@ -1,25 +1,20 @@ |
|||
const fs = require("fs") |
|||
const jimp = require("jimp") |
|||
const fsPromises = fs.promises |
|||
|
|||
const FORMATS = { |
|||
IMAGES: ["png", "jpg", "jpeg", "gif", "bmp", "tiff"], |
|||
} |
|||
|
|||
function processImage(file) { |
|||
// this will overwrite the temp file
|
|||
return jimp.read(file.path).then(img => { |
|||
return img.resize(300, jimp.AUTO).write(file.outputPath) |
|||
return img.resize(300, jimp.AUTO).write(file.path) |
|||
}) |
|||
} |
|||
|
|||
async function process(file) { |
|||
if (FORMATS.IMAGES.includes(file.extension.toLowerCase())) { |
|||
await processImage(file) |
|||
return file |
|||
} |
|||
|
|||
// No processing required
|
|||
await fsPromises.copyFile(file.path, file.outputPath) |
|||
return file |
|||
} |
|||
|
|||
@ -0,0 +1,243 @@ |
|||
const sanitize = require("sanitize-s3-objectkey") |
|||
const AWS = require("aws-sdk") |
|||
const stream = require("stream") |
|||
const fetch = require("node-fetch") |
|||
const tar = require("tar-fs") |
|||
const zlib = require("zlib") |
|||
const { promisify } = require("util") |
|||
const { join } = require("path") |
|||
const fs = require("fs") |
|||
const { budibaseTempDir } = require("../budibaseDir") |
|||
const env = require("../../environment") |
|||
const { ObjectStoreBuckets } = require("../../constants") |
|||
const uuid = require("uuid/v4") |
|||
|
|||
const streamPipeline = promisify(stream.pipeline) |
|||
// use this as a temporary store of buckets that are being created
|
|||
const STATE = { |
|||
bucketCreationPromises: {}, |
|||
} |
|||
|
|||
const CONTENT_TYPE_MAP = { |
|||
html: "text/html", |
|||
css: "text/css", |
|||
js: "application/javascript", |
|||
} |
|||
const STRING_CONTENT_TYPES = [ |
|||
CONTENT_TYPE_MAP.html, |
|||
CONTENT_TYPE_MAP.css, |
|||
CONTENT_TYPE_MAP.js, |
|||
] |
|||
|
|||
function publicPolicy(bucketName) { |
|||
return { |
|||
Version: "2012-10-17", |
|||
Statement: [ |
|||
{ |
|||
Effect: "Allow", |
|||
Principal: { |
|||
AWS: ["*"], |
|||
}, |
|||
Action: "s3:GetObject", |
|||
Resource: [`arn:aws:s3:::${bucketName}/*`], |
|||
}, |
|||
], |
|||
} |
|||
} |
|||
|
|||
const PUBLIC_BUCKETS = [ObjectStoreBuckets.APPS] |
|||
|
|||
/** |
|||
* Gets a connection to the object store using the S3 SDK. |
|||
* @param {string} bucket the name of the bucket which blobs will be uploaded/retrieved from. |
|||
* @return {Object} an S3 object store object, check S3 Nodejs SDK for usage. |
|||
* @constructor |
|||
*/ |
|||
exports.ObjectStore = bucket => { |
|||
if (env.SELF_HOSTED) { |
|||
AWS.config.update({ |
|||
accessKeyId: env.MINIO_ACCESS_KEY, |
|||
secretAccessKey: env.MINIO_SECRET_KEY, |
|||
}) |
|||
} |
|||
const config = { |
|||
s3ForcePathStyle: true, |
|||
signatureVersion: "v4", |
|||
params: { |
|||
Bucket: bucket, |
|||
}, |
|||
} |
|||
if (env.MINIO_URL) { |
|||
config.endpoint = env.MINIO_URL |
|||
} |
|||
return new AWS.S3(config) |
|||
} |
|||
|
|||
/** |
|||
* Given an object store and a bucket name this will make sure the bucket exists, |
|||
* if it does not exist then it will create it. |
|||
*/ |
|||
exports.makeSureBucketExists = async (client, bucketName) => { |
|||
try { |
|||
await client |
|||
.headBucket({ |
|||
Bucket: bucketName, |
|||
}) |
|||
.promise() |
|||
} catch (err) { |
|||
const promises = STATE.bucketCreationPromises |
|||
if (promises[bucketName]) { |
|||
await promises[bucketName] |
|||
} else if (err.statusCode === 404) { |
|||
// bucket doesn't exist create it
|
|||
promises[bucketName] = client |
|||
.createBucket({ |
|||
Bucket: bucketName, |
|||
}) |
|||
.promise() |
|||
await promises[bucketName] |
|||
delete promises[bucketName] |
|||
// public buckets are quite hidden in the system, make sure
|
|||
// no bucket is set accidentally
|
|||
if (PUBLIC_BUCKETS.includes(bucketName)) { |
|||
await client |
|||
.putBucketPolicy({ |
|||
Bucket: bucketName, |
|||
Policy: JSON.stringify(publicPolicy(bucketName)), |
|||
}) |
|||
.promise() |
|||
} |
|||
} else { |
|||
throw err |
|||
} |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* Uploads the contents of a file given the required parameters, useful when |
|||
* temp files in use (for example file uploaded as an attachment). |
|||
*/ |
|||
exports.upload = async ({ bucket, filename, path, type, metadata }) => { |
|||
const extension = [...filename.split(".")].pop() |
|||
const fileBytes = fs.readFileSync(path) |
|||
|
|||
const objectStore = exports.ObjectStore(bucket) |
|||
await exports.makeSureBucketExists(objectStore, bucket) |
|||
|
|||
const config = { |
|||
// windows file paths need to be converted to forward slashes for s3
|
|||
Key: sanitize(filename).replace(/\\/g, "/"), |
|||
Body: fileBytes, |
|||
ContentType: type || CONTENT_TYPE_MAP[extension.toLowerCase()], |
|||
} |
|||
if (metadata) { |
|||
config.Metadata = metadata |
|||
} |
|||
return objectStore.upload(config).promise() |
|||
} |
|||
|
|||
/** |
|||
* Similar to the upload function but can be used to send a file stream |
|||
* through to the object store. |
|||
*/ |
|||
exports.streamUpload = async (bucket, filename, stream) => { |
|||
const objectStore = exports.ObjectStore(bucket) |
|||
await exports.makeSureBucketExists(objectStore, bucket) |
|||
|
|||
const params = { |
|||
Bucket: bucket, |
|||
Key: sanitize(filename).replace(/\\/g, "/"), |
|||
Body: stream, |
|||
} |
|||
return objectStore.upload(params).promise() |
|||
} |
|||
|
|||
/** |
|||
* retrieves the contents of a file from the object store, if it is a known content type it |
|||
* will be converted, otherwise it will be returned as a buffer stream. |
|||
*/ |
|||
exports.retrieve = async (bucket, filepath) => { |
|||
const objectStore = exports.ObjectStore(bucket) |
|||
const params = { |
|||
Bucket: bucket, |
|||
Key: sanitize(filepath).replace(/\\/g, "/"), |
|||
} |
|||
const response = await objectStore.getObject(params).promise() |
|||
// currently these are all strings
|
|||
if (STRING_CONTENT_TYPES.includes(response.ContentType)) { |
|||
return response.Body.toString("utf8") |
|||
} else { |
|||
return response.Body |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* Same as retrieval function but puts to a temporary file. |
|||
*/ |
|||
exports.retrieveToTmp = async (bucket, filepath) => { |
|||
const data = await exports.retrieve(bucket, filepath) |
|||
const outputPath = join(budibaseTempDir(), uuid()) |
|||
fs.writeFileSync(outputPath, data) |
|||
return outputPath |
|||
} |
|||
|
|||
exports.deleteFolder = async (bucket, folder) => { |
|||
const client = exports.ObjectStore(bucket) |
|||
const listParams = { |
|||
Bucket: bucket, |
|||
Prefix: folder, |
|||
} |
|||
|
|||
let response = await client.listObjects(listParams).promise() |
|||
if (response.Contents.length === 0) { |
|||
return |
|||
} |
|||
const deleteParams = { |
|||
Bucket: bucket, |
|||
Delete: { |
|||
Objects: [], |
|||
}, |
|||
} |
|||
|
|||
response.Contents.forEach(content => { |
|||
deleteParams.Delete.Objects.push({ Key: content.Key }) |
|||
}) |
|||
|
|||
response = await client.deleteObjects(deleteParams).promise() |
|||
// can only empty 1000 items at once
|
|||
if (response.Deleted.length === 1000) { |
|||
return exports.deleteFolder(bucket, folder) |
|||
} |
|||
} |
|||
|
|||
exports.uploadDirectory = async (bucket, localPath, bucketPath) => { |
|||
let uploads = [] |
|||
const files = fs.readdirSync(localPath, { withFileTypes: true }) |
|||
for (let file of files) { |
|||
const path = join(bucketPath, file.name) |
|||
const local = join(localPath, file.name) |
|||
if (file.isDirectory()) { |
|||
uploads.push(exports.uploadDirectory(bucket, local, path)) |
|||
} else { |
|||
uploads.push( |
|||
exports.streamUpload(bucket, path, fs.createReadStream(local)) |
|||
) |
|||
} |
|||
} |
|||
await Promise.all(uploads) |
|||
} |
|||
|
|||
exports.downloadTarball = async (url, bucket, path) => { |
|||
const response = await fetch(url) |
|||
if (!response.ok) { |
|||
throw new Error(`unexpected response ${response.statusText}`) |
|||
} |
|||
|
|||
const tmpPath = join(budibaseTempDir(), path) |
|||
await streamPipeline(response.body, zlib.Unzip(), tar.extract(tmpPath)) |
|||
if (!env.isTest()) { |
|||
await exports.uploadDirectory(bucket, tmpPath, path) |
|||
} |
|||
// return the temporary path incase there is a use for it
|
|||
return tmpPath |
|||
} |
|||
@ -1,37 +0,0 @@ |
|||
const { existsSync, readFile, writeFile, ensureDir } = require("fs-extra") |
|||
const { join, resolve } = require("./centralPath") |
|||
const { processString } = require("@budibase/string-templates") |
|||
const uuid = require("uuid") |
|||
|
|||
module.exports = async opts => { |
|||
await ensureDir(opts.dir) |
|||
await setCouchDbUrl(opts) |
|||
|
|||
// need an env file
|
|||
await createDevEnvFile(opts) |
|||
} |
|||
|
|||
const setCouchDbUrl = async opts => { |
|||
if (!opts.couchDbUrl) { |
|||
const dataDir = join(opts.dir, ".data") |
|||
await ensureDir(dataDir) |
|||
opts.couchDbUrl = |
|||
dataDir + (dataDir.endsWith("/") || dataDir.endsWith("\\") ? "" : "/") |
|||
} |
|||
} |
|||
|
|||
const createDevEnvFile = async opts => { |
|||
const destConfigFile = join(opts.dir, "./.env") |
|||
let createConfig = !existsSync(destConfigFile) || opts.quiet |
|||
if (createConfig) { |
|||
const template = await readFile( |
|||
resolve(__dirname, "..", "..", ".env.template"), |
|||
{ |
|||
encoding: "utf8", |
|||
} |
|||
) |
|||
opts.cookieKey1 = opts.cookieKey1 || uuid.v4() |
|||
const config = await processString(template, opts) |
|||
await writeFile(destConfigFile, config, { flag: "w+" }) |
|||
} |
|||
} |
|||
@ -1,81 +0,0 @@ |
|||
const fs = require("fs-extra") |
|||
const { join } = require("./centralPath") |
|||
const os = require("os") |
|||
const fetch = require("node-fetch") |
|||
const stream = require("stream") |
|||
const tar = require("tar-fs") |
|||
const zlib = require("zlib") |
|||
const { promisify } = require("util") |
|||
const streamPipeline = promisify(stream.pipeline) |
|||
const { budibaseAppsDir } = require("./budibaseDir") |
|||
const env = require("../environment") |
|||
const CouchDB = require("../db") |
|||
|
|||
const DEFAULT_TEMPLATES_BUCKET = |
|||
"prod-budi-templates.s3-eu-west-1.amazonaws.com" |
|||
|
|||
exports.getLocalTemplates = function() { |
|||
const templatesDir = join(os.homedir(), ".budibase", "templates", "app") |
|||
const templateObj = { app: {} } |
|||
fs.ensureDirSync(templatesDir) |
|||
const templateNames = fs.readdirSync(templatesDir) |
|||
for (let name of templateNames) { |
|||
templateObj.app[name] = { |
|||
name, |
|||
category: "local", |
|||
description: "local template", |
|||
type: "app", |
|||
key: `app/${name}`, |
|||
} |
|||
} |
|||
return templateObj |
|||
} |
|||
|
|||
// can't really test this, downloading is just not something we should do in a behavioural test
|
|||
/* istanbul ignore next */ |
|||
exports.downloadTemplate = async function(type, name) { |
|||
const dirName = join(budibaseAppsDir(), "templates", type, name) |
|||
if (env.LOCAL_TEMPLATES) { |
|||
return dirName |
|||
} |
|||
const templateUrl = `https://${DEFAULT_TEMPLATES_BUCKET}/templates/${type}/${name}.tar.gz` |
|||
const response = await fetch(templateUrl) |
|||
|
|||
if (!response.ok) { |
|||
throw new Error( |
|||
`Error downloading template ${type}:${name}: ${response.statusText}` |
|||
) |
|||
} |
|||
|
|||
// stream the response, unzip and extract
|
|||
await streamPipeline( |
|||
response.body, |
|||
zlib.Unzip(), |
|||
tar.extract(join(budibaseAppsDir(), "templates", type)) |
|||
) |
|||
|
|||
return dirName |
|||
} |
|||
|
|||
async function performDump({ dir, appId, name = "dump.txt" }) { |
|||
const writeStream = fs.createWriteStream(join(dir, name)) |
|||
// perform couch dump
|
|||
const instanceDb = new CouchDB(appId) |
|||
await instanceDb.dump(writeStream, {}) |
|||
} |
|||
|
|||
exports.performDump = performDump |
|||
|
|||
exports.exportTemplateFromApp = async function({ templateName, appId }) { |
|||
// Copy frontend files
|
|||
const templatesDir = join( |
|||
budibaseAppsDir(), |
|||
"templates", |
|||
"app", |
|||
templateName, |
|||
"db" |
|||
) |
|||
fs.ensureDirSync(templatesDir) |
|||
await performDump({ dir: templatesDir, appId }) |
|||
return templatesDir |
|||
} |
|||
File diff suppressed because it is too large
Loading…
Reference in new issue