mirror of https://github.com/Budibase/budibase.git
Martin McKeaveney
5 years ago
85 changed files with 3148 additions and 1132 deletions
@ -0,0 +1 @@ |
|||
hosting.properties |
|||
@ -0,0 +1,16 @@ |
|||
version: "3" |
|||
|
|||
services: |
|||
app-service: |
|||
build: ./server |
|||
volumes: |
|||
- ./server:/app |
|||
environment: |
|||
SELF_HOSTED: 1 |
|||
PORT: 4002 |
|||
|
|||
worker-service: |
|||
build: ./worker |
|||
environment: |
|||
SELF_HOSTED: 1, |
|||
PORT: 4003 |
|||
@ -0,0 +1 @@ |
|||
../../packages/server/ |
|||
@ -0,0 +1 @@ |
|||
../../packages/worker/ |
|||
@ -0,0 +1,90 @@ |
|||
version: "3" |
|||
|
|||
services: |
|||
app-service: |
|||
image: budibase/budibase-apps |
|||
ports: |
|||
- "${APP_PORT}:4002" |
|||
environment: |
|||
SELF_HOSTED: 1 |
|||
COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984 |
|||
BUDIBASE_ENVIRONMENT: ${BUDIBASE_ENVIRONMENT} |
|||
LOGO_URL: ${LOGO_URL} |
|||
PORT: 4002 |
|||
JWT_SECRET: ${JWT_SECRET} |
|||
depends_on: |
|||
- worker-service |
|||
|
|||
worker-service: |
|||
image: budibase/budibase-worker |
|||
ports: |
|||
- "${WORKER_PORT}:4003" |
|||
environment: |
|||
SELF_HOSTED: 1, |
|||
DEPLOYMENT_API_KEY: ${WORKER_API_KEY} |
|||
PORT: 4003 |
|||
MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} |
|||
MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} |
|||
RAW_MINIO_URL: http://minio-service:9000 |
|||
COUCH_DB_USERNAME: ${COUCH_DB_USER} |
|||
COUCH_DB_PASSWORD: ${COUCH_DB_PASSWORD} |
|||
RAW_COUCH_DB_URL: http://couchdb-service:5984 |
|||
SELF_HOST_KEY: ${HOSTING_KEY} |
|||
depends_on: |
|||
- minio-service |
|||
- couch-init |
|||
|
|||
minio-service: |
|||
image: minio/minio |
|||
volumes: |
|||
- minio_data:/data |
|||
ports: |
|||
- "${MINIO_PORT}:9000" |
|||
environment: |
|||
MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} |
|||
MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} |
|||
command: server /data |
|||
healthcheck: |
|||
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] |
|||
interval: 30s |
|||
timeout: 20s |
|||
retries: 3 |
|||
|
|||
proxy-service: |
|||
image: envoyproxy/envoy:v1.16-latest |
|||
volumes: |
|||
- ./envoy.yaml:/etc/envoy/envoy.yaml |
|||
ports: |
|||
- "${MAIN_PORT}:10000" |
|||
- "9901:9901" |
|||
depends_on: |
|||
- minio-service |
|||
- worker-service |
|||
- app-service |
|||
- couchdb-service |
|||
|
|||
couchdb-service: |
|||
image: apache/couchdb:3.0 |
|||
environment: |
|||
- COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} |
|||
- COUCHDB_USER=${COUCH_DB_USER} |
|||
ports: |
|||
- "${COUCH_DB_PORT}:5984" |
|||
- "4369:4369" |
|||
- "9100:9100" |
|||
volumes: |
|||
- couchdb_data:/couchdb |
|||
|
|||
couch-init: |
|||
image: curlimages/curl |
|||
environment: |
|||
PUT_CALL: "curl -u ${COUCH_DB_USER}:${COUCH_DB_PASSWORD} -X PUT couchdb-service:5984" |
|||
depends_on: |
|||
- couchdb-service |
|||
command: ["sh","-c","sleep 10 && $${PUT_CALL}/_users && $${PUT_CALL}/_replicator; fg;"] |
|||
|
|||
volumes: |
|||
couchdb_data: |
|||
driver: local |
|||
minio_data: |
|||
driver: local |
|||
@ -0,0 +1,104 @@ |
|||
static_resources: |
|||
listeners: |
|||
- name: main_listener |
|||
address: |
|||
socket_address: { address: 0.0.0.0, port_value: 10000 } |
|||
filter_chains: |
|||
- filters: |
|||
- name: envoy.filters.network.http_connection_manager |
|||
typed_config: |
|||
"@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager |
|||
stat_prefix: ingress |
|||
codec_type: auto |
|||
route_config: |
|||
name: local_route |
|||
virtual_hosts: |
|||
- name: local_services |
|||
domains: ["*"] |
|||
routes: |
|||
- match: { prefix: "/app/" } |
|||
route: |
|||
cluster: app-service |
|||
prefix_rewrite: "/" |
|||
|
|||
# special case for when API requests are made, can just forward, not to minio |
|||
- match: { prefix: "/api/" } |
|||
route: |
|||
cluster: app-service |
|||
|
|||
- match: { prefix: "/worker/" } |
|||
route: |
|||
cluster: worker-service |
|||
prefix_rewrite: "/" |
|||
|
|||
- match: { prefix: "/db/" } |
|||
route: |
|||
cluster: couchdb-service |
|||
prefix_rewrite: "/" |
|||
|
|||
# minio is on the default route because this works |
|||
# best, minio + AWS SDK doesn't handle path proxy |
|||
- match: { prefix: "/" } |
|||
route: |
|||
cluster: minio-service |
|||
|
|||
http_filters: |
|||
- name: envoy.filters.http.router |
|||
|
|||
clusters: |
|||
- name: app-service |
|||
connect_timeout: 0.25s |
|||
type: strict_dns |
|||
lb_policy: round_robin |
|||
load_assignment: |
|||
cluster_name: app-service |
|||
endpoints: |
|||
- lb_endpoints: |
|||
- endpoint: |
|||
address: |
|||
socket_address: |
|||
address: app-service |
|||
port_value: 4002 |
|||
|
|||
- name: minio-service |
|||
connect_timeout: 0.25s |
|||
type: strict_dns |
|||
lb_policy: round_robin |
|||
load_assignment: |
|||
cluster_name: minio-service |
|||
endpoints: |
|||
- lb_endpoints: |
|||
- endpoint: |
|||
address: |
|||
socket_address: |
|||
address: minio-service |
|||
port_value: 9000 |
|||
|
|||
- name: worker-service |
|||
connect_timeout: 0.25s |
|||
type: strict_dns |
|||
lb_policy: round_robin |
|||
load_assignment: |
|||
cluster_name: worker-service |
|||
endpoints: |
|||
- lb_endpoints: |
|||
- endpoint: |
|||
address: |
|||
socket_address: |
|||
address: worker-service |
|||
port_value: 4003 |
|||
|
|||
- name: couchdb-service |
|||
connect_timeout: 0.25s |
|||
type: strict_dns |
|||
lb_policy: round_robin |
|||
load_assignment: |
|||
cluster_name: couchdb-service |
|||
endpoints: |
|||
- lb_endpoints: |
|||
- endpoint: |
|||
address: |
|||
socket_address: |
|||
address: couchdb-service |
|||
port_value: 5984 |
|||
|
|||
@ -0,0 +1,25 @@ |
|||
# Use the main port in the builder for your self hosting URL, e.g. localhost:10000 |
|||
MAIN_PORT=10000 |
|||
|
|||
# Use this password when configuring your self hosting settings |
|||
# This should be updated |
|||
HOSTING_KEY=budibase |
|||
|
|||
# This section contains customisation options |
|||
LOGO_URL=https://logoipsum.com/logo/logo-15.svg |
|||
|
|||
# This section contains all secrets pertaining to the system |
|||
# These should be updated |
|||
JWT_SECRET=testsecret |
|||
MINIO_ACCESS_KEY=budibase |
|||
MINIO_SECRET_KEY=budibase |
|||
COUCH_DB_PASSWORD=budibase |
|||
COUCH_DB_USER=budibase |
|||
WORKER_API_KEY=budibase |
|||
|
|||
# This section contains variables that do not need to be altered under normal circumstances |
|||
APP_PORT=4002 |
|||
WORKER_PORT=4003 |
|||
MINIO_PORT=4004 |
|||
COUCH_DB_PORT=4005 |
|||
BUDIBASE_ENVIRONMENT=PRODUCTION |
|||
@ -0,0 +1,4 @@ |
|||
#!/bin/bash |
|||
sudo curl -L "https://github.com/docker/compose/releases/download/1.27.4/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose |
|||
sudo chmod +x /usr/local/bin/docker-compose |
|||
sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose |
|||
@ -0,0 +1,6 @@ |
|||
#!/bin/bash |
|||
echo "**** WARNING - not for production environments ****" |
|||
# warning this is a convience script, for production installations install docker |
|||
# properly for your environment! |
|||
curl -fsSL https://get.docker.com -o get-docker.sh |
|||
sudo sh get-docker.sh |
|||
@ -0,0 +1,9 @@ |
|||
#!/bin/bash |
|||
pushd ../../build |
|||
docker-compose build --force app-service |
|||
docker-compose build --force worker-service |
|||
docker tag build_app-service budibase/budibase-apps:latest |
|||
docker push budibase/budibase-apps |
|||
docker tag build_worker-service budibase/budibase-worker:latest |
|||
docker push budibase/budibase-worker |
|||
popd |
|||
@ -0,0 +1,2 @@ |
|||
#!/bin/bash |
|||
docker-compose --env-file hosting.properties up |
|||
@ -0,0 +1,20 @@ |
|||
#!/bin/bash |
|||
|
|||
function dockerInstalled { |
|||
echo "Checking docker installation..." |
|||
if [ ! -x "$(command -v docker)" ]; then |
|||
echo "Please install docker to continue" |
|||
exit -1 |
|||
fi |
|||
} |
|||
|
|||
dockerInstalled |
|||
|
|||
source "${BASH_SOURCE%/*}/hosting.properties" |
|||
|
|||
opts="-e MINIO_ACCESS_KEY=$minio_access_key -e MINIO_SECRET_KEY=$minio_secret_key" |
|||
if [ -n "$minio_secret_key_old" ] && [ -n "$minio_access_key_old" ]; then |
|||
opts="$opts -e MINIO_SECRET_KEY_OLD=$minio_secret_key_old -e MINIO_ACCESS_KEY_OLD=$minio_access_key_old" |
|||
fi |
|||
|
|||
docker run -p $minio_port:$minio_port $opts -v /mnt/data:/data minio/minio server /data |
|||
@ -0,0 +1,38 @@ |
|||
import { writable } from "svelte/store" |
|||
import api from "../api" |
|||
|
|||
const INITIAL_BACKEND_UI_STATE = { |
|||
hostingInfo: {}, |
|||
appUrl: "", |
|||
} |
|||
|
|||
export const getHostingStore = () => { |
|||
const store = writable({ ...INITIAL_BACKEND_UI_STATE }) |
|||
store.actions = { |
|||
fetch: async () => { |
|||
const responses = await Promise.all([ |
|||
api.get("/api/hosting/"), |
|||
api.get("/api/hosting/urls"), |
|||
]) |
|||
const [info, urls] = await Promise.all(responses.map(resp => resp.json())) |
|||
store.update(state => { |
|||
state.hostingInfo = info |
|||
state.appUrl = urls.app |
|||
return state |
|||
}) |
|||
return info |
|||
}, |
|||
save: async hostingInfo => { |
|||
const response = await api.post("/api/hosting", hostingInfo) |
|||
const revision = (await response.json()).rev |
|||
store.update(state => { |
|||
state.hostingInfo = { |
|||
...hostingInfo, |
|||
_rev: revision, |
|||
} |
|||
return state |
|||
}) |
|||
}, |
|||
} |
|||
return store |
|||
} |
|||
@ -1,41 +0,0 @@ |
|||
<script> |
|||
import { |
|||
DropdownMenu, |
|||
TextButton as Button, |
|||
Icon, |
|||
Modal, |
|||
ModalContent, |
|||
} from "@budibase/bbui" |
|||
import { backendUiStore } from "builderStore" |
|||
import api from "builderStore/api" |
|||
import EditIntegrationConfig from "../modals/EditIntegrationConfig.svelte" |
|||
|
|||
export let table |
|||
|
|||
let modal |
|||
|
|||
// TODO: revisit |
|||
async function saveTable() { |
|||
const SAVE_TABLE_URL = `/api/tables` |
|||
const response = await api.post(SAVE_TABLE_URL, table) |
|||
const savedTable = await response.json() |
|||
await backendUiStore.actions.tables.fetch() |
|||
backendUiStore.actions.tables.select(savedTable) |
|||
} |
|||
</script> |
|||
|
|||
<div> |
|||
<Button text small on:click={modal.show}> |
|||
<Icon name="edit" /> |
|||
Configure Schema |
|||
</Button> |
|||
</div> |
|||
<Modal bind:this={modal}> |
|||
<ModalContent |
|||
confirmText="Save" |
|||
cancelText="Cancel" |
|||
onConfirm={saveTable} |
|||
title={'Datasource Configuration'}> |
|||
<EditIntegrationConfig onClosed={modal.hide} bind:table /> |
|||
</ModalContent> |
|||
</Modal> |
|||
@ -1,129 +0,0 @@ |
|||
<script> |
|||
import { |
|||
Select, |
|||
Button, |
|||
Input, |
|||
TextArea, |
|||
Heading, |
|||
Spacer, |
|||
} from "@budibase/bbui" |
|||
import { notifier } from "builderStore/store/notifications" |
|||
import { FIELDS } from "constants/backend" |
|||
import { backendUiStore } from "builderStore" |
|||
import * as api from "../api" |
|||
|
|||
export let table |
|||
|
|||
let smartSchemaRow |
|||
let fields = Object.keys(table.schema).map(field => ({ |
|||
name: field, |
|||
type: table.schema[field].type.toUpperCase(), |
|||
})) |
|||
|
|||
$: { |
|||
const schema = {} |
|||
for (let field of fields) { |
|||
if (!field.name) continue |
|||
schema[field.name] = FIELDS[field.type] |
|||
} |
|||
table.schema = schema |
|||
} |
|||
|
|||
function newField() { |
|||
fields = [...fields, {}] |
|||
} |
|||
|
|||
function deleteField(idx) { |
|||
fields.splice(idx, 1) |
|||
fields = fields |
|||
} |
|||
|
|||
async function smartSchema() { |
|||
try { |
|||
const rows = await api.fetchDataForView($backendUiStore.selectedView) |
|||
const first = rows[0] |
|||
smartSchemaRow = first |
|||
fields = Object.keys(first).map(key => ({ |
|||
// TODO: Smarter type mapping |
|||
name: key, |
|||
type: "STRING", |
|||
})) |
|||
} catch (err) { |
|||
notifier.danger("Error determining schema. Please enter fields manually.") |
|||
} |
|||
} |
|||
</script> |
|||
|
|||
<section> |
|||
<div class="config"> |
|||
<h6>Schema</h6> |
|||
{#if smartSchemaRow} |
|||
<pre>{JSON.stringify(smartSchemaRow, undefined, 2)}</pre> |
|||
{/if} |
|||
{#each fields as field, idx} |
|||
<div class="field"> |
|||
<Input thin type={'text'} bind:value={field.name} /> |
|||
<Select secondary thin bind:value={field.type}> |
|||
<option value={''}>Select an option</option> |
|||
<option value={'STRING'}>Text</option> |
|||
<option value={'NUMBER'}>Number</option> |
|||
<option value={'BOOLEAN'}>Boolean</option> |
|||
<option value={'DATETIME'}>Datetime</option> |
|||
</Select> |
|||
<i |
|||
class="ri-close-circle-line delete" |
|||
on:click={() => deleteField(idx)} /> |
|||
</div> |
|||
{/each} |
|||
<Button thin secondary on:click={newField}>Add Field</Button> |
|||
<Button thin primary on:click={smartSchema}>Smart Schema</Button> |
|||
</div> |
|||
|
|||
<div class="config"> |
|||
<h6>Datasource</h6> |
|||
{#each Object.keys(table.integration) as configKey} |
|||
{#if configKey === 'query'} |
|||
<TextArea |
|||
thin |
|||
label={configKey} |
|||
bind:value={table.integration[configKey]} /> |
|||
{:else} |
|||
<Input |
|||
thin |
|||
type={configKey.type} |
|||
label={configKey} |
|||
bind:value={table.integration[configKey]} /> |
|||
{/if} |
|||
<Spacer small /> |
|||
{/each} |
|||
</div> |
|||
</section> |
|||
|
|||
<style> |
|||
.field { |
|||
display: grid; |
|||
grid-gap: 10px; |
|||
grid-template-columns: 1fr 1fr 50px; |
|||
margin-bottom: var(--spacing-m); |
|||
} |
|||
|
|||
h6 { |
|||
font-family: var(--font-sans); |
|||
font-weight: 600; |
|||
text-rendering: var(--text-render); |
|||
color: var(--ink); |
|||
font-size: var(--heading-font-size-xs); |
|||
color: var(--ink); |
|||
margin-bottom: var(--spacing-m); |
|||
margin-top: var(--spacing-l); |
|||
} |
|||
|
|||
.config { |
|||
margin-bottom: var(--spacing-s); |
|||
} |
|||
|
|||
.delete { |
|||
align-self: center; |
|||
cursor: pointer; |
|||
} |
|||
</style> |
|||
@ -0,0 +1,55 @@ |
|||
<script> |
|||
import { Label, DropdownMenu } from "@budibase/bbui" |
|||
import ThemeEditor from "./ThemeEditor.svelte" |
|||
|
|||
let anchor |
|||
let popover |
|||
</script> |
|||
|
|||
<div class="topnavitemright" on:click={popover.show} bind:this={anchor}> |
|||
<i class="ri-paint-fill" /> |
|||
</div> |
|||
<div class="dropdown"> |
|||
<DropdownMenu bind:this={popover} {anchor} align="right"> |
|||
<div class="content"> |
|||
<Label extraSmall grey>Theme</Label> |
|||
<ThemeEditor /> |
|||
</div> |
|||
</DropdownMenu> |
|||
</div> |
|||
|
|||
<style> |
|||
.dropdown { |
|||
z-index: 2; |
|||
} |
|||
|
|||
i { |
|||
font-size: 18px; |
|||
color: var(--grey-7); |
|||
} |
|||
.topnavitemright { |
|||
cursor: pointer; |
|||
color: var(--grey-7); |
|||
margin: 0 12px 0 0; |
|||
font-weight: 500; |
|||
font-size: 1rem; |
|||
display: flex; |
|||
flex-direction: row; |
|||
justify-content: center; |
|||
align-items: center; |
|||
height: 24px; |
|||
width: 24px; |
|||
} |
|||
.topnavitemright:hover i { |
|||
color: var(--ink); |
|||
} |
|||
|
|||
h5 { |
|||
margin: 0; |
|||
font-weight: 500; |
|||
} |
|||
|
|||
.content { |
|||
padding: var(--spacing-xl); |
|||
} |
|||
</style> |
|||
@ -0,0 +1,32 @@ |
|||
<script> |
|||
import { TextButton as Button, Modal } from "@budibase/bbui" |
|||
import BuilderSettingsModal from "./BuilderSettingsModal.svelte" |
|||
|
|||
let modal |
|||
</script> |
|||
|
|||
<div> |
|||
<Button text on:click={modal.show}> |
|||
<i class="ri-settings-3-fill" /> |
|||
<p>Settings</p> |
|||
</Button> |
|||
</div> |
|||
<Modal bind:this={modal} width="30%"> |
|||
<BuilderSettingsModal /> |
|||
</Modal> |
|||
|
|||
<style> |
|||
div i { |
|||
font-size: 26px; |
|||
color: var(--grey-7); |
|||
margin-left: 12px; |
|||
} |
|||
|
|||
div p { |
|||
font-family: var(--font-sans); |
|||
font-size: var(--font-size-s); |
|||
color: var(--ink); |
|||
font-weight: 400; |
|||
margin: 0 0 0 12px; |
|||
} |
|||
</style> |
|||
@ -0,0 +1,72 @@ |
|||
<script> |
|||
import { notifier } from "builderStore/store/notifications" |
|||
import { hostingStore } from "builderStore" |
|||
import { Input, ModalContent, Toggle } from "@budibase/bbui" |
|||
import ThemeEditor from "components/settings/ThemeEditor.svelte" |
|||
import analytics from "analytics" |
|||
import { onMount } from "svelte" |
|||
|
|||
let hostingInfo |
|||
let selfhosted = false |
|||
|
|||
async function save() { |
|||
hostingInfo.type = selfhosted ? "self" : "cloud" |
|||
if (!selfhosted && hostingInfo._rev) { |
|||
hostingInfo = { |
|||
type: hostingInfo.type, |
|||
_id: hostingInfo._id, |
|||
_rev: hostingInfo._rev, |
|||
} |
|||
} |
|||
try { |
|||
await hostingStore.actions.save(hostingInfo) |
|||
notifier.success(`Settings saved.`) |
|||
} catch (err) { |
|||
notifier.danger(`Failed to update builder settings.`) |
|||
} |
|||
} |
|||
|
|||
function updateSelfHosting(event) { |
|||
if (hostingInfo.type === "cloud" && event.target.checked) { |
|||
hostingInfo.hostingUrl = "localhost:10000" |
|||
hostingInfo.useHttps = false |
|||
hostingInfo.selfHostKey = "budibase" |
|||
} |
|||
} |
|||
|
|||
onMount(async () => { |
|||
hostingInfo = await hostingStore.actions.fetch() |
|||
selfhosted = hostingInfo.type === "self" |
|||
}) |
|||
</script> |
|||
|
|||
<ModalContent title="Builder settings" confirmText="Save" onConfirm={save}> |
|||
<h5>Theme</h5> |
|||
<ThemeEditor /> |
|||
<h5>Hosting</h5> |
|||
<p> |
|||
This section contains settings that relate to the deployment and hosting of |
|||
apps made in this builder. |
|||
</p> |
|||
<Toggle |
|||
thin |
|||
text="Self hosted" |
|||
on:change={updateSelfHosting} |
|||
bind:checked={selfhosted} /> |
|||
{#if selfhosted} |
|||
<Input bind:value={hostingInfo.hostingUrl} label="Hosting URL" /> |
|||
<Input bind:value={hostingInfo.selfHostKey} label="Hosting Key" /> |
|||
<Toggle thin text="HTTPS" bind:checked={hostingInfo.useHttps} /> |
|||
{/if} |
|||
</ModalContent> |
|||
|
|||
<style> |
|||
h5 { |
|||
margin: 0; |
|||
font-size: 14px; |
|||
} |
|||
p { |
|||
margin: 0; |
|||
font-size: 12px; |
|||
} |
|||
</style> |
|||
@ -1,47 +0,0 @@ |
|||
const COOKIE_SEPARATOR = ";" |
|||
const APP_PREFIX = "app_" |
|||
const KEY_VALUE_SPLIT = "=" |
|||
|
|||
function confirmAppId(possibleAppId) { |
|||
return possibleAppId && possibleAppId.startsWith(APP_PREFIX) |
|||
? possibleAppId |
|||
: undefined |
|||
} |
|||
|
|||
function tryGetFromCookie({ cookies }) { |
|||
if (!cookies) { |
|||
return undefined |
|||
} |
|||
const cookie = cookies |
|||
.split(COOKIE_SEPARATOR) |
|||
.find(cookie => cookie.trim().startsWith("budibase:currentapp")) |
|||
let appId |
|||
if (cookie && cookie.split(KEY_VALUE_SPLIT).length === 2) { |
|||
appId = cookie.split("=")[1] |
|||
} |
|||
return confirmAppId(appId) |
|||
} |
|||
|
|||
function tryGetFromPath() { |
|||
const appId = location.pathname.split("/")[1] |
|||
return confirmAppId(appId) |
|||
} |
|||
|
|||
function tryGetFromSubdomain() { |
|||
const parts = window.location.host.split(".") |
|||
const appId = parts[1] ? parts[0] : undefined |
|||
return confirmAppId(appId) |
|||
} |
|||
|
|||
export const getAppId = (cookies = window.document.cookie) => { |
|||
const functions = [tryGetFromSubdomain, tryGetFromPath, tryGetFromCookie] |
|||
// try getting the app Id in order
|
|||
let appId |
|||
for (let func of functions) { |
|||
appId = func({ cookies }) |
|||
if (appId) { |
|||
break |
|||
} |
|||
} |
|||
return appId |
|||
} |
|||
@ -1,56 +0,0 @@ |
|||
// THIS will create API Keys and App Ids input in a local Dynamo instance if it is running
|
|||
const dynamoClient = require("../src/db/dynamoClient") |
|||
const env = require("../src/environment") |
|||
|
|||
if (process.argv[2] == null || process.argv[3] == null) { |
|||
console.error( |
|||
"Inputs incorrect format, was expecting: node createApiKeyAndAppId.js <API_KEY> <APP_ID>" |
|||
) |
|||
process.exit(-1) |
|||
} |
|||
|
|||
const FAKE_STRING = "fakestring" |
|||
|
|||
// set fake credentials for local dynamo to actually work
|
|||
env._set("AWS_ACCESS_KEY_ID", "KEY_ID") |
|||
env._set("AWS_SECRET_ACCESS_KEY", "SECRET_KEY") |
|||
dynamoClient.init("http://localhost:8333") |
|||
|
|||
async function run() { |
|||
await dynamoClient.apiKeyTable.put({ |
|||
item: { |
|||
pk: process.argv[2], |
|||
accountId: FAKE_STRING, |
|||
trackingId: FAKE_STRING, |
|||
quotaReset: Date.now() + 2592000000, |
|||
usageQuota: { |
|||
automationRuns: 0, |
|||
rows: 0, |
|||
storage: 0, |
|||
users: 0, |
|||
views: 0, |
|||
}, |
|||
usageLimits: { |
|||
automationRuns: 10, |
|||
rows: 10, |
|||
storage: 1000, |
|||
users: 10, |
|||
views: 10, |
|||
}, |
|||
}, |
|||
}) |
|||
await dynamoClient.apiKeyTable.put({ |
|||
item: { |
|||
pk: process.argv[3], |
|||
apiKey: process.argv[2], |
|||
}, |
|||
}) |
|||
} |
|||
|
|||
run() |
|||
.then(() => { |
|||
console.log("Rows should have been created.") |
|||
}) |
|||
.catch(err => { |
|||
console.error("Cannot create rows - " + err) |
|||
}) |
|||
@ -0,0 +1,88 @@ |
|||
const { getAppQuota } = require("./quota") |
|||
const env = require("../../../environment") |
|||
const newid = require("../../../db/newid") |
|||
|
|||
/** |
|||
* This is used to pass around information about the deployment that is occurring |
|||
*/ |
|||
class Deployment { |
|||
constructor(appId, id = null) { |
|||
this.appId = appId |
|||
this._id = id || newid() |
|||
} |
|||
|
|||
// purely so that we can do quota stuff outside the main deployment context
|
|||
async init() { |
|||
if (!env.SELF_HOSTED) { |
|||
this.setQuota(await getAppQuota(this.appId)) |
|||
} |
|||
} |
|||
|
|||
setQuota(quota) { |
|||
if (!quota) { |
|||
return |
|||
} |
|||
this.quota = quota |
|||
} |
|||
|
|||
getQuota() { |
|||
return this.quota |
|||
} |
|||
|
|||
getAppId() { |
|||
return this.appId |
|||
} |
|||
|
|||
setVerification(verification) { |
|||
if (!verification) { |
|||
return |
|||
} |
|||
this.verification = verification |
|||
if (this.verification.quota) { |
|||
this.quota = this.verification.quota |
|||
} |
|||
} |
|||
|
|||
getVerification() { |
|||
return this.verification |
|||
} |
|||
|
|||
setStatus(status, err = null) { |
|||
this.status = status |
|||
if (err) { |
|||
this.err = err |
|||
} |
|||
} |
|||
|
|||
fromJSON(json) { |
|||
if (json.verification) { |
|||
this.setVerification(json.verification) |
|||
} |
|||
if (json.quota) { |
|||
this.setQuota(json.quota) |
|||
} |
|||
if (json.status) { |
|||
this.setStatus(json.status, json.err) |
|||
} |
|||
} |
|||
|
|||
getJSON() { |
|||
const obj = { |
|||
_id: this._id, |
|||
appId: this.appId, |
|||
status: this.status, |
|||
} |
|||
if (this.err) { |
|||
obj.err = this.err |
|||
} |
|||
if (this.verification && this.verification.cfDistribution) { |
|||
obj.cfDistribution = this.verification.cfDistribution |
|||
} |
|||
if (this.quota) { |
|||
obj.quota = this.quota |
|||
} |
|||
return obj |
|||
} |
|||
} |
|||
|
|||
module.exports = Deployment |
|||
@ -1,189 +0,0 @@ |
|||
const fs = require("fs") |
|||
const { join } = require("../../../utilities/centralPath") |
|||
const AWS = require("aws-sdk") |
|||
const fetch = require("node-fetch") |
|||
const sanitize = require("sanitize-s3-objectkey") |
|||
const { budibaseAppsDir } = require("../../../utilities/budibaseDir") |
|||
const PouchDB = require("../../../db") |
|||
const env = require("../../../environment") |
|||
|
|||
/** |
|||
* Finalises the deployment, updating the quota for the user API key |
|||
* The verification process returns the levels to update to. |
|||
* Calls the "deployment-success" lambda. |
|||
* @param {object} quota The usage quota levels returned from the verifyDeploy |
|||
* @returns {Promise<object>} The usage has been updated against the user API key. |
|||
*/ |
|||
exports.updateDeploymentQuota = async function(quota) { |
|||
const DEPLOYMENT_SUCCESS_URL = |
|||
env.DEPLOYMENT_CREDENTIALS_URL + "deploy/success" |
|||
|
|||
const response = await fetch(DEPLOYMENT_SUCCESS_URL, { |
|||
method: "POST", |
|||
body: JSON.stringify({ |
|||
apiKey: env.BUDIBASE_API_KEY, |
|||
quota, |
|||
}), |
|||
headers: { |
|||
"Content-Type": "application/json", |
|||
Accept: "application/json", |
|||
}, |
|||
}) |
|||
|
|||
if (response.status !== 200) { |
|||
throw new Error(`Error updating deployment quota for API Key`) |
|||
} |
|||
|
|||
return await response.json() |
|||
} |
|||
|
|||
/** |
|||
* Verifies the users API key and |
|||
* Verifies that the deployment fits within the quota of the user |
|||
* Links to the "check-api-key" lambda. |
|||
* @param {String} appId - appId being deployed |
|||
* @param {String} appId - appId being deployed |
|||
* @param {quota} quota - current quota being changed with this application |
|||
*/ |
|||
exports.verifyDeployment = async function({ appId, quota }) { |
|||
const response = await fetch(env.DEPLOYMENT_CREDENTIALS_URL, { |
|||
method: "POST", |
|||
body: JSON.stringify({ |
|||
apiKey: env.BUDIBASE_API_KEY, |
|||
appId, |
|||
quota, |
|||
}), |
|||
}) |
|||
|
|||
const json = await response.json() |
|||
if (json.errors) { |
|||
throw new Error(json.errors) |
|||
} |
|||
|
|||
if (response.status !== 200) { |
|||
throw new Error( |
|||
`Error fetching temporary credentials for api key: ${env.BUDIBASE_API_KEY}` |
|||
) |
|||
} |
|||
|
|||
// set credentials here, means any time we're verified we're ready to go
|
|||
if (json.credentials) { |
|||
AWS.config.update({ |
|||
accessKeyId: json.credentials.AccessKeyId, |
|||
secretAccessKey: json.credentials.SecretAccessKey, |
|||
sessionToken: json.credentials.SessionToken, |
|||
}) |
|||
} |
|||
|
|||
return json |
|||
} |
|||
|
|||
const CONTENT_TYPE_MAP = { |
|||
html: "text/html", |
|||
css: "text/css", |
|||
js: "application/javascript", |
|||
} |
|||
|
|||
/** |
|||
* Recursively walk a directory tree and execute a callback on all files. |
|||
* @param {String} dirPath - Directory to traverse |
|||
* @param {Function} callback - callback to execute on files |
|||
*/ |
|||
function walkDir(dirPath, callback) { |
|||
for (let filename of fs.readdirSync(dirPath)) { |
|||
const filePath = `${dirPath}/${filename}` |
|||
const stat = fs.lstatSync(filePath) |
|||
|
|||
if (stat.isFile()) { |
|||
callback(filePath) |
|||
} else { |
|||
walkDir(filePath, callback) |
|||
} |
|||
} |
|||
} |
|||
|
|||
async function prepareUploadForS3({ s3Key, metadata, s3, file }) { |
|||
const extension = [...file.name.split(".")].pop() |
|||
const fileBytes = fs.readFileSync(file.path) |
|||
|
|||
const upload = await s3 |
|||
.upload({ |
|||
// windows filepaths need to be converted to forward slashes for s3
|
|||
Key: sanitize(s3Key).replace(/\\/g, "/"), |
|||
Body: fileBytes, |
|||
ContentType: file.type || CONTENT_TYPE_MAP[extension.toLowerCase()], |
|||
Metadata: metadata, |
|||
}) |
|||
.promise() |
|||
|
|||
return { |
|||
size: file.size, |
|||
name: file.name, |
|||
extension, |
|||
url: upload.Location, |
|||
key: upload.Key, |
|||
} |
|||
} |
|||
|
|||
exports.prepareUploadForS3 = prepareUploadForS3 |
|||
|
|||
exports.uploadAppAssets = async function({ appId, bucket, accountId }) { |
|||
const s3 = new AWS.S3({ |
|||
params: { |
|||
Bucket: bucket, |
|||
}, |
|||
}) |
|||
|
|||
const appAssetsPath = join(budibaseAppsDir(), appId, "public") |
|||
|
|||
let uploads = [] |
|||
|
|||
// Upload HTML and JS of the web app
|
|||
walkDir(appAssetsPath, function(filePath) { |
|||
const filePathParts = filePath.split("/") |
|||
const appAssetUpload = prepareUploadForS3({ |
|||
file: { |
|||
path: filePath, |
|||
name: filePathParts.pop(), |
|||
}, |
|||
s3Key: filePath.replace(appAssetsPath, `assets/${appId}`), |
|||
s3, |
|||
metadata: { accountId }, |
|||
}) |
|||
uploads.push(appAssetUpload) |
|||
}) |
|||
|
|||
// Upload file attachments
|
|||
const db = new PouchDB(appId) |
|||
let fileUploads |
|||
try { |
|||
fileUploads = await db.get("_local/fileuploads") |
|||
} catch (err) { |
|||
fileUploads = { _id: "_local/fileuploads", uploads: [] } |
|||
} |
|||
|
|||
for (let file of fileUploads.uploads) { |
|||
if (file.uploaded) continue |
|||
|
|||
const attachmentUpload = prepareUploadForS3({ |
|||
file, |
|||
s3Key: `assets/${appId}/attachments/${file.processedFileName}`, |
|||
s3, |
|||
metadata: { accountId }, |
|||
}) |
|||
|
|||
uploads.push(attachmentUpload) |
|||
|
|||
// mark file as uploaded
|
|||
file.uploaded = true |
|||
} |
|||
|
|||
db.put(fileUploads) |
|||
|
|||
try { |
|||
return await Promise.all(uploads) |
|||
} catch (err) { |
|||
console.error("Error uploading budibase app assets to s3", err) |
|||
throw err |
|||
} |
|||
} |
|||
@ -0,0 +1,85 @@ |
|||
const AWS = require("aws-sdk") |
|||
const fetch = require("node-fetch") |
|||
const env = require("../../../environment") |
|||
const { |
|||
deployToObjectStore, |
|||
performReplication, |
|||
fetchCredentials, |
|||
} = require("./utils") |
|||
|
|||
/** |
|||
* Verifies the users API key and |
|||
* Verifies that the deployment fits within the quota of the user |
|||
* Links to the "check-api-key" lambda. |
|||
* @param {object} deployment - information about the active deployment, including the appId and quota. |
|||
*/ |
|||
exports.preDeployment = async function(deployment) { |
|||
const json = await fetchCredentials(env.DEPLOYMENT_CREDENTIALS_URL, { |
|||
apiKey: env.BUDIBASE_API_KEY, |
|||
appId: deployment.getAppId(), |
|||
quota: deployment.getQuota(), |
|||
}) |
|||
|
|||
// set credentials here, means any time we're verified we're ready to go
|
|||
if (json.credentials) { |
|||
AWS.config.update({ |
|||
accessKeyId: json.credentials.AccessKeyId, |
|||
secretAccessKey: json.credentials.SecretAccessKey, |
|||
sessionToken: json.credentials.SessionToken, |
|||
}) |
|||
} |
|||
|
|||
return json |
|||
} |
|||
|
|||
/** |
|||
* Finalises the deployment, updating the quota for the user API key |
|||
* The verification process returns the levels to update to. |
|||
* Calls the "deployment-success" lambda. |
|||
* @param {object} deployment information about the active deployment, including the quota info. |
|||
* @returns {Promise<object>} The usage has been updated against the user API key. |
|||
*/ |
|||
exports.postDeployment = async function(deployment) { |
|||
const DEPLOYMENT_SUCCESS_URL = |
|||
env.DEPLOYMENT_CREDENTIALS_URL + "deploy/success" |
|||
|
|||
const response = await fetch(DEPLOYMENT_SUCCESS_URL, { |
|||
method: "POST", |
|||
body: JSON.stringify({ |
|||
apiKey: env.BUDIBASE_API_KEY, |
|||
quota: deployment.getQuota(), |
|||
}), |
|||
headers: { |
|||
"Content-Type": "application/json", |
|||
Accept: "application/json", |
|||
}, |
|||
}) |
|||
|
|||
if (response.status !== 200) { |
|||
throw new Error(`Error updating deployment quota for API Key`) |
|||
} |
|||
|
|||
return await response.json() |
|||
} |
|||
|
|||
exports.deploy = async function(deployment) { |
|||
const appId = deployment.getAppId() |
|||
const { bucket, accountId } = deployment.getVerification() |
|||
const metadata = { accountId } |
|||
const s3Client = new AWS.S3({ |
|||
params: { |
|||
Bucket: bucket, |
|||
}, |
|||
}) |
|||
await deployToObjectStore(appId, s3Client, metadata) |
|||
} |
|||
|
|||
exports.replicateDb = async function(deployment) { |
|||
const appId = deployment.getAppId() |
|||
const verification = deployment.getVerification() |
|||
return performReplication( |
|||
appId, |
|||
verification.couchDbSession, |
|||
env.DEPLOYMENT_DB_URL |
|||
) |
|||
} |
|||
@ -0,0 +1,27 @@ |
|||
const PouchDB = require("../../../db") |
|||
const { DocumentTypes, SEPARATOR, UNICODE_MAX } = require("../../../db/utils") |
|||
|
|||
exports.getAppQuota = async function(appId) { |
|||
const db = new PouchDB(appId) |
|||
|
|||
const rows = await db.allDocs({ |
|||
startkey: DocumentTypes.ROW + SEPARATOR, |
|||
endkey: DocumentTypes.ROW + SEPARATOR + UNICODE_MAX, |
|||
}) |
|||
|
|||
const users = await db.allDocs({ |
|||
startkey: DocumentTypes.USER + SEPARATOR, |
|||
endkey: DocumentTypes.USER + SEPARATOR + UNICODE_MAX, |
|||
}) |
|||
|
|||
const existingRows = rows.rows.length |
|||
const existingUsers = users.rows.length |
|||
|
|||
const designDoc = await db.get("_design/database") |
|||
|
|||
return { |
|||
rows: existingRows, |
|||
users: existingUsers, |
|||
views: Object.keys(designDoc.views).length, |
|||
} |
|||
} |
|||
@ -0,0 +1,69 @@ |
|||
const AWS = require("aws-sdk") |
|||
const { |
|||
deployToObjectStore, |
|||
performReplication, |
|||
fetchCredentials, |
|||
} = require("./utils") |
|||
const { |
|||
getWorkerUrl, |
|||
getCouchUrl, |
|||
getMinioUrl, |
|||
getSelfHostKey, |
|||
} = require("../../../utilities/builder/hosting") |
|||
|
|||
exports.preDeployment = async function() { |
|||
const url = `${await getWorkerUrl()}/api/deploy` |
|||
try { |
|||
const json = await fetchCredentials(url, { |
|||
selfHostKey: await getSelfHostKey(), |
|||
}) |
|||
|
|||
// response contains:
|
|||
// couchDbSession, bucket, objectStoreSession
|
|||
|
|||
// set credentials here, means any time we're verified we're ready to go
|
|||
if (json.objectStoreSession) { |
|||
AWS.config.update({ |
|||
accessKeyId: json.objectStoreSession.accessKeyId, |
|||
secretAccessKey: json.objectStoreSession.secretAccessKey, |
|||
}) |
|||
} |
|||
|
|||
return json |
|||
} catch (err) { |
|||
throw { |
|||
message: "Unauthorised to deploy, check self hosting key", |
|||
status: 401, |
|||
} |
|||
} |
|||
} |
|||
|
|||
exports.postDeployment = async function() { |
|||
// we don't actively need to do anything after deployment in self hosting
|
|||
} |
|||
|
|||
exports.deploy = async function(deployment) { |
|||
const appId = deployment.getAppId() |
|||
const verification = deployment.getVerification() |
|||
const objClient = new AWS.S3({ |
|||
endpoint: await getMinioUrl(), |
|||
s3ForcePathStyle: true, // needed with minio?
|
|||
signatureVersion: "v4", |
|||
params: { |
|||
Bucket: verification.bucket, |
|||
}, |
|||
}) |
|||
// no metadata, aws has account ID in metadata
|
|||
const metadata = {} |
|||
await deployToObjectStore(appId, objClient, metadata) |
|||
} |
|||
|
|||
exports.replicateDb = async function(deployment) { |
|||
const appId = deployment.getAppId() |
|||
const verification = deployment.getVerification() |
|||
return performReplication( |
|||
appId, |
|||
verification.couchDbSession, |
|||
await getCouchUrl() |
|||
) |
|||
} |
|||
@ -0,0 +1,131 @@ |
|||
const fs = require("fs") |
|||
const sanitize = require("sanitize-s3-objectkey") |
|||
const { walkDir } = require("../../../utilities") |
|||
const { join } = require("../../../utilities/centralPath") |
|||
const { budibaseAppsDir } = require("../../../utilities/budibaseDir") |
|||
const fetch = require("node-fetch") |
|||
const PouchDB = require("../../../db") |
|||
const CouchDB = require("pouchdb") |
|||
|
|||
const CONTENT_TYPE_MAP = { |
|||
html: "text/html", |
|||
css: "text/css", |
|||
js: "application/javascript", |
|||
} |
|||
|
|||
exports.fetchCredentials = async function(url, body) { |
|||
const response = await fetch(url, { |
|||
method: "POST", |
|||
body: JSON.stringify(body), |
|||
headers: { "Content-Type": "application/json" }, |
|||
}) |
|||
|
|||
const json = await response.json() |
|||
if (json.errors) { |
|||
throw new Error(json.errors) |
|||
} |
|||
|
|||
if (response.status !== 200) { |
|||
throw new Error( |
|||
`Error fetching temporary credentials: ${JSON.stringify(json)}` |
|||
) |
|||
} |
|||
|
|||
return json |
|||
} |
|||
|
|||
exports.prepareUpload = async function({ s3Key, metadata, client, file }) { |
|||
const extension = [...file.name.split(".")].pop() |
|||
const fileBytes = fs.readFileSync(file.path) |
|||
|
|||
const upload = await client |
|||
.upload({ |
|||
// windows file paths need to be converted to forward slashes for s3
|
|||
Key: sanitize(s3Key).replace(/\\/g, "/"), |
|||
Body: fileBytes, |
|||
ContentType: file.type || CONTENT_TYPE_MAP[extension.toLowerCase()], |
|||
Metadata: metadata, |
|||
}) |
|||
.promise() |
|||
|
|||
return { |
|||
size: file.size, |
|||
name: file.name, |
|||
extension, |
|||
url: upload.Location, |
|||
key: upload.Key, |
|||
} |
|||
} |
|||
|
|||
exports.deployToObjectStore = async function(appId, objectClient, metadata) { |
|||
const appAssetsPath = join(budibaseAppsDir(), appId, "public") |
|||
|
|||
let uploads = [] |
|||
|
|||
// Upload HTML, CSS and JS for each page of the web app
|
|||
walkDir(appAssetsPath, function(filePath) { |
|||
const filePathParts = filePath.split("/") |
|||
const appAssetUpload = exports.prepareUpload({ |
|||
file: { |
|||
path: filePath, |
|||
name: filePathParts.pop(), |
|||
}, |
|||
s3Key: filePath.replace(appAssetsPath, `assets/${appId}`), |
|||
client: objectClient, |
|||
metadata, |
|||
}) |
|||
uploads.push(appAssetUpload) |
|||
}) |
|||
|
|||
// Upload file attachments
|
|||
const db = new PouchDB(appId) |
|||
let fileUploads |
|||
try { |
|||
fileUploads = await db.get("_local/fileuploads") |
|||
} catch (err) { |
|||
fileUploads = { _id: "_local/fileuploads", uploads: [] } |
|||
} |
|||
|
|||
for (let file of fileUploads.uploads) { |
|||
if (file.uploaded) continue |
|||
|
|||
const attachmentUpload = exports.prepareUpload({ |
|||
file, |
|||
s3Key: `assets/${appId}/attachments/${file.processedFileName}`, |
|||
client: objectClient, |
|||
metadata, |
|||
}) |
|||
|
|||
uploads.push(attachmentUpload) |
|||
|
|||
// mark file as uploaded
|
|||
file.uploaded = true |
|||
} |
|||
|
|||
db.put(fileUploads) |
|||
|
|||
try { |
|||
return await Promise.all(uploads) |
|||
} catch (err) { |
|||
console.error("Error uploading budibase app assets to s3", err) |
|||
throw err |
|||
} |
|||
} |
|||
|
|||
exports.performReplication = (appId, session, dbUrl) => { |
|||
return new Promise((resolve, reject) => { |
|||
const local = new PouchDB(appId) |
|||
|
|||
const remote = new CouchDB(`${dbUrl}/${appId}`, { |
|||
fetch: function(url, opts) { |
|||
opts.headers.set("Cookie", `${session};`) |
|||
return CouchDB.fetch(url, opts) |
|||
}, |
|||
}) |
|||
|
|||
const replication = local.sync(remote) |
|||
|
|||
replication.on("complete", () => resolve()) |
|||
replication.on("error", err => reject(err)) |
|||
}) |
|||
} |
|||
@ -0,0 +1,39 @@ |
|||
const CouchDB = require("../../db") |
|||
const { BUILDER_CONFIG_DB, HOSTING_DOC } = require("../../constants") |
|||
const { |
|||
getHostingInfo, |
|||
HostingTypes, |
|||
getAppUrl, |
|||
} = require("../../utilities/builder/hosting") |
|||
|
|||
exports.fetchInfo = async ctx => { |
|||
ctx.body = { |
|||
types: Object.values(HostingTypes), |
|||
} |
|||
} |
|||
|
|||
exports.save = async ctx => { |
|||
const db = new CouchDB(BUILDER_CONFIG_DB) |
|||
const { type } = ctx.request.body |
|||
if (type === HostingTypes.CLOUD && ctx.request.body._rev) { |
|||
ctx.body = await db.remove({ |
|||
...ctx.request.body, |
|||
_id: HOSTING_DOC, |
|||
}) |
|||
} else { |
|||
ctx.body = await db.put({ |
|||
...ctx.request.body, |
|||
_id: HOSTING_DOC, |
|||
}) |
|||
} |
|||
} |
|||
|
|||
exports.fetch = async ctx => { |
|||
ctx.body = await getHostingInfo() |
|||
} |
|||
|
|||
exports.fetchUrls = async ctx => { |
|||
ctx.body = { |
|||
app: await getAppUrl(ctx.appId), |
|||
} |
|||
} |
|||
@ -0,0 +1,14 @@ |
|||
const Router = require("@koa/router") |
|||
const controller = require("../controllers/hosting") |
|||
const authorized = require("../../middleware/authorized") |
|||
const { BUILDER } = require("../../utilities/security/permissions") |
|||
|
|||
const router = Router() |
|||
|
|||
router |
|||
.get("/api/hosting/info", authorized(BUILDER), controller.fetchInfo) |
|||
.get("/api/hosting/urls", authorized(BUILDER), controller.fetchUrls) |
|||
.get("/api/hosting", authorized(BUILDER), controller.fetch) |
|||
.post("/api/hosting", authorized(BUILDER), controller.save) |
|||
|
|||
module.exports = router |
|||
@ -0,0 +1,7 @@ |
|||
### Self hosting |
|||
This directory contains utilities that are needed for self hosted platforms to operate. |
|||
These will mostly be utilities, necessary to the operation of the server e.g. storing self |
|||
hosting specific options and attributes to CouchDB. |
|||
|
|||
All the internal operations should be exposed through the `index.js` so importing |
|||
the self host directory should give you everything you need. |
|||
@ -0,0 +1,44 @@ |
|||
const CouchDB = require("../db") |
|||
const env = require("../environment") |
|||
const newid = require("../db/newid") |
|||
|
|||
const SELF_HOST_DB = "self-host-db" |
|||
const SELF_HOST_DOC = "self-host-info" |
|||
|
|||
async function createSelfHostDB(db) { |
|||
await db.put({ |
|||
_id: "_design/database", |
|||
views: {}, |
|||
}) |
|||
const selfHostInfo = { |
|||
_id: SELF_HOST_DOC, |
|||
apiKeyId: newid(), |
|||
} |
|||
await db.put(selfHostInfo) |
|||
return selfHostInfo |
|||
} |
|||
|
|||
exports.init = async () => { |
|||
if (!env.SELF_HOSTED) { |
|||
return |
|||
} |
|||
const db = new CouchDB(SELF_HOST_DB) |
|||
try { |
|||
await db.get(SELF_HOST_DOC) |
|||
} catch (err) { |
|||
// failed to retrieve
|
|||
if (err.status === 404) { |
|||
await createSelfHostDB(db) |
|||
} |
|||
} |
|||
} |
|||
|
|||
exports.getSelfHostInfo = async () => { |
|||
const db = new CouchDB(SELF_HOST_DB) |
|||
return db.get(SELF_HOST_DOC) |
|||
} |
|||
|
|||
exports.getSelfHostAPIKey = async () => { |
|||
const info = await exports.getSelfHostInfo() |
|||
return info ? info.apiKeyId : null |
|||
} |
|||
@ -0,0 +1,83 @@ |
|||
const CouchDB = require("../../db") |
|||
const { BUILDER_CONFIG_DB, HOSTING_DOC } = require("../../constants") |
|||
|
|||
const PROD_HOSTING_URL = "app.budi.live" |
|||
|
|||
function getProtocol(hostingInfo) { |
|||
return hostingInfo.useHttps ? "https://" : "http://" |
|||
} |
|||
|
|||
async function getURLWithPath(pathIfSelfHosted) { |
|||
const hostingInfo = await exports.getHostingInfo() |
|||
const protocol = getProtocol(hostingInfo) |
|||
const path = |
|||
hostingInfo.type === exports.HostingTypes.SELF ? pathIfSelfHosted : "" |
|||
return `${protocol}${hostingInfo.hostingUrl}${path}` |
|||
} |
|||
|
|||
exports.HostingTypes = { |
|||
CLOUD: "cloud", |
|||
SELF: "self", |
|||
} |
|||
|
|||
exports.getHostingInfo = async () => { |
|||
const db = new CouchDB(BUILDER_CONFIG_DB) |
|||
let doc |
|||
try { |
|||
doc = await db.get(HOSTING_DOC) |
|||
} catch (err) { |
|||
// don't write this doc, want to be able to update these default props
|
|||
// for our servers with a new release without needing to worry about state of
|
|||
// PouchDB in peoples installations
|
|||
doc = { |
|||
_id: HOSTING_DOC, |
|||
type: exports.HostingTypes.CLOUD, |
|||
hostingUrl: PROD_HOSTING_URL, |
|||
selfHostKey: "", |
|||
templatesUrl: "prod-budi-templates.s3-eu-west-1.amazonaws.com", |
|||
useHttps: true, |
|||
} |
|||
} |
|||
return doc |
|||
} |
|||
|
|||
exports.getAppUrl = async appId => { |
|||
const hostingInfo = await exports.getHostingInfo() |
|||
const protocol = getProtocol(hostingInfo) |
|||
let url |
|||
if (hostingInfo.type === exports.HostingTypes.CLOUD) { |
|||
url = `${protocol}${appId}.${hostingInfo.hostingUrl}` |
|||
} else { |
|||
url = `${protocol}${hostingInfo.hostingUrl}/app` |
|||
} |
|||
return url |
|||
} |
|||
|
|||
exports.getWorkerUrl = async () => { |
|||
return getURLWithPath("/worker") |
|||
} |
|||
|
|||
exports.getMinioUrl = async () => { |
|||
return getURLWithPath("/") |
|||
} |
|||
|
|||
exports.getCouchUrl = async () => { |
|||
return getURLWithPath("/db") |
|||
} |
|||
|
|||
exports.getSelfHostKey = async () => { |
|||
const hostingInfo = await exports.getHostingInfo() |
|||
return hostingInfo.selfHostKey |
|||
} |
|||
|
|||
exports.getTemplatesUrl = async (appId, type, name) => { |
|||
const hostingInfo = await exports.getHostingInfo() |
|||
const protocol = getProtocol(hostingInfo) |
|||
let path |
|||
if (type && name) { |
|||
path = `templates/type/${name}.tar.gz` |
|||
} else { |
|||
path = "manifest.json" |
|||
} |
|||
return `${protocol}${hostingInfo.templatesUrl}/${path}` |
|||
} |
|||
@ -0,0 +1,23 @@ |
|||
const { apiKeyTable } = require("../../db/dynamoClient") |
|||
const env = require("../../environment") |
|||
const { getSelfHostAPIKey } = require("../../selfhost") |
|||
|
|||
/** |
|||
* This file purely exists so that we can centralise all logic pertaining to API keys, as their usage differs |
|||
* in our Cloud environment versus self hosted. |
|||
*/ |
|||
|
|||
exports.isAPIKeyValid = async apiKeyId => { |
|||
if (env.CLOUD && !env.SELF_HOSTED) { |
|||
let apiKeyInfo = await apiKeyTable.get({ |
|||
primary: apiKeyId, |
|||
}) |
|||
return apiKeyInfo != null |
|||
} |
|||
if (env.SELF_HOSTED) { |
|||
const selfHostKey = await getSelfHostAPIKey() |
|||
// if the api key supplied is correct then return structure similar
|
|||
return apiKeyId === selfHostKey ? { pk: apiKeyId } : null |
|||
} |
|||
return false |
|||
} |
|||
File diff suppressed because it is too large
@ -0,0 +1,2 @@ |
|||
node_modules/ |
|||
.env |
|||
@ -0,0 +1,15 @@ |
|||
FROM node:12-alpine |
|||
|
|||
WORKDIR /app |
|||
|
|||
# copy files and install dependencies |
|||
COPY . ./ |
|||
RUN yarn |
|||
|
|||
EXPOSE 4001 |
|||
|
|||
# have to add node environment production after install |
|||
# due to this causing yarn to stop installing dev dependencies |
|||
# which are actually needed to get this environment up and running |
|||
ENV NODE_ENV=production |
|||
CMD ["yarn", "run:docker"] |
|||
@ -0,0 +1,34 @@ |
|||
{ |
|||
"name": "@budibase/deployment", |
|||
"email": "hi@budibase.com", |
|||
"version": "0.3.8", |
|||
"description": "Budibase Deployment Server", |
|||
"main": "src/index.js", |
|||
"repository": { |
|||
"type": "git", |
|||
"url": "https://github.com/Budibase/budibase.git" |
|||
}, |
|||
"keywords": [ |
|||
"budibase" |
|||
], |
|||
"scripts": { |
|||
"run:docker": "node src/index.js" |
|||
}, |
|||
"author": "Budibase", |
|||
"license": "AGPL-3.0-or-later", |
|||
"dependencies": { |
|||
"@koa/router": "^8.0.0", |
|||
"aws-sdk": "^2.811.0", |
|||
"got": "^11.8.1", |
|||
"joi": "^17.2.1", |
|||
"koa": "^2.7.0", |
|||
"koa-body": "^4.2.0", |
|||
"koa-compress": "^4.0.1", |
|||
"koa-pino-logger": "^3.0.0", |
|||
"koa-send": "^5.0.0", |
|||
"koa-session": "^5.12.0", |
|||
"koa-static": "^5.0.0", |
|||
"pino-pretty": "^4.0.0", |
|||
"server-destroy": "^1.0.1" |
|||
} |
|||
} |
|||
@ -0,0 +1,92 @@ |
|||
const env = require("../../environment") |
|||
const got = require("got") |
|||
const AWS = require("aws-sdk") |
|||
|
|||
const APP_BUCKET = "app-assets" |
|||
// this doesn't matter in self host
|
|||
const REGION = "eu-west-1" |
|||
const PUBLIC_READ_POLICY = { |
|||
Version: "2012-10-17", |
|||
Statement: [ |
|||
{ |
|||
Effect: "Allow", |
|||
Principal: { |
|||
AWS: ["*"], |
|||
}, |
|||
Action: "s3:GetObject", |
|||
Resource: [`arn:aws:s3:::${APP_BUCKET}/*`], |
|||
}, |
|||
], |
|||
} |
|||
|
|||
async function getCouchSession() { |
|||
// fetch session token for the api user
|
|||
const session = await got.post(`${env.RAW_COUCH_DB_URL}/_session`, { |
|||
responseType: "json", |
|||
credentials: "include", |
|||
json: { |
|||
username: env.COUCH_DB_USERNAME, |
|||
password: env.COUCH_DB_PASSWORD, |
|||
}, |
|||
}) |
|||
|
|||
const cookie = session.headers["set-cookie"][0] |
|||
// Get the session cookie value only
|
|||
return cookie.split(";")[0] |
|||
} |
|||
|
|||
async function getMinioSession() { |
|||
AWS.config.update({ |
|||
accessKeyId: env.MINIO_ACCESS_KEY, |
|||
secretAccessKey: env.MINIO_SECRET_KEY, |
|||
}) |
|||
|
|||
// make sure the bucket exists
|
|||
const objClient = new AWS.S3({ |
|||
endpoint: env.RAW_MINIO_URL, |
|||
region: REGION, |
|||
s3ForcePathStyle: true, // needed with minio?
|
|||
params: { |
|||
Bucket: APP_BUCKET, |
|||
}, |
|||
}) |
|||
// make sure the bucket exists
|
|||
try { |
|||
await objClient |
|||
.headBucket({ |
|||
Bucket: APP_BUCKET, |
|||
}) |
|||
.promise() |
|||
} catch (err) { |
|||
// bucket doesn't exist create it
|
|||
if (err.statusCode === 404) { |
|||
await objClient |
|||
.createBucket({ |
|||
Bucket: APP_BUCKET, |
|||
}) |
|||
.promise() |
|||
} else { |
|||
throw err |
|||
} |
|||
} |
|||
// always make sure policy is correct
|
|||
await objClient |
|||
.putBucketPolicy({ |
|||
Bucket: APP_BUCKET, |
|||
Policy: JSON.stringify(PUBLIC_READ_POLICY), |
|||
}) |
|||
.promise() |
|||
// Ideally want to send back some pre-signed URLs for files that are to be uploaded
|
|||
return { |
|||
accessKeyId: env.MINIO_ACCESS_KEY, |
|||
secretAccessKey: env.MINIO_SECRET_KEY, |
|||
} |
|||
} |
|||
|
|||
exports.deploy = async ctx => { |
|||
ctx.body = { |
|||
couchDbSession: await getCouchSession(), |
|||
bucket: APP_BUCKET, |
|||
objectStoreSession: await getMinioSession(), |
|||
} |
|||
} |
|||
@ -0,0 +1,45 @@ |
|||
const Router = require("@koa/router") |
|||
const compress = require("koa-compress") |
|||
const zlib = require("zlib") |
|||
const { routes } = require("./routes") |
|||
|
|||
const router = new Router() |
|||
|
|||
router |
|||
.use( |
|||
compress({ |
|||
threshold: 2048, |
|||
gzip: { |
|||
flush: zlib.Z_SYNC_FLUSH, |
|||
}, |
|||
deflate: { |
|||
flush: zlib.Z_SYNC_FLUSH, |
|||
}, |
|||
br: false, |
|||
}) |
|||
) |
|||
.use("/health", ctx => (ctx.status = 200)) |
|||
|
|||
// error handling middleware
|
|||
router.use(async (ctx, next) => { |
|||
try { |
|||
await next() |
|||
} catch (err) { |
|||
ctx.log.error(err) |
|||
ctx.status = err.status || err.statusCode || 500 |
|||
ctx.body = { |
|||
message: err.message, |
|||
status: ctx.status, |
|||
} |
|||
} |
|||
}) |
|||
|
|||
router.get("/health", ctx => (ctx.status = 200)) |
|||
|
|||
// authenticated routes
|
|||
for (let route of routes) { |
|||
router.use(route.routes()) |
|||
router.use(route.allowedMethods()) |
|||
} |
|||
|
|||
module.exports = router |
|||
@ -0,0 +1,9 @@ |
|||
const Router = require("@koa/router") |
|||
const controller = require("../controllers/deploy") |
|||
const checkKey = require("../../middleware/check-key") |
|||
|
|||
const router = Router() |
|||
|
|||
router.post("/api/deploy", checkKey, controller.deploy) |
|||
|
|||
module.exports = router |
|||
@ -0,0 +1,3 @@ |
|||
const deployRoutes = require("./deploy") |
|||
|
|||
exports.routes = [deployRoutes] |
|||
@ -0,0 +1,18 @@ |
|||
module.exports = { |
|||
SELF_HOSTED: process.env.SELF_HOSTED, |
|||
WORKER_API_KEY: process.env.WORKER_API_KEY, |
|||
PORT: process.env.PORT, |
|||
MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY, |
|||
MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY, |
|||
COUCH_DB_USERNAME: process.env.COUCH_DB_USERNAME, |
|||
COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD, |
|||
RAW_COUCH_DB_URL: process.env.RAW_COUCH_DB_URL, |
|||
RAW_MINIO_URL: process.env.RAW_MINIO_URL, |
|||
COUCH_DB_PORT: process.env.COUCH_DB_PORT, |
|||
MINIO_PORT: process.env.MINIO_PORT, |
|||
SELF_HOST_KEY: process.env.SELF_HOST_KEY, |
|||
_set(key, value) { |
|||
process.env[key] = value |
|||
module.exports[key] = value |
|||
}, |
|||
} |
|||
@ -0,0 +1,48 @@ |
|||
const Koa = require("koa") |
|||
const destroyable = require("server-destroy") |
|||
const koaBody = require("koa-body") |
|||
const logger = require("koa-pino-logger") |
|||
const http = require("http") |
|||
const api = require("./api") |
|||
const env = require("./environment") |
|||
|
|||
const app = new Koa() |
|||
|
|||
if (!env.SELF_HOSTED) { |
|||
throw "Currently this service only supports use in self hosting" |
|||
} |
|||
|
|||
// set up top level koa middleware
|
|||
app.use(koaBody({ multipart: true })) |
|||
|
|||
app.use( |
|||
logger({ |
|||
prettyPrint: { |
|||
levelFirst: true, |
|||
}, |
|||
level: env.LOG_LEVEL || "error", |
|||
}) |
|||
) |
|||
|
|||
// api routes
|
|||
app.use(api.routes()) |
|||
|
|||
const server = http.createServer(app.callback()) |
|||
destroyable(server) |
|||
|
|||
server.on("close", () => console.log("Server Closed")) |
|||
|
|||
module.exports = server.listen(env.PORT || 4002, async () => { |
|||
console.log(`Worker running on ${JSON.stringify(server.address())}`) |
|||
}) |
|||
|
|||
process.on("uncaughtException", err => { |
|||
console.error(err) |
|||
server.close() |
|||
server.destroy() |
|||
}) |
|||
|
|||
process.on("SIGTERM", () => { |
|||
server.close() |
|||
server.destroy() |
|||
}) |
|||
@ -0,0 +1,12 @@ |
|||
const env = require("../environment") |
|||
|
|||
module.exports = async (ctx, next) => { |
|||
if ( |
|||
!ctx.request.body.selfHostKey || |
|||
env.SELF_HOST_KEY !== ctx.request.body.selfHostKey |
|||
) { |
|||
ctx.throw(401, "Deployment unauthorised") |
|||
} else { |
|||
await next() |
|||
} |
|||
} |
|||
File diff suppressed because it is too large
Loading…
Reference in new issue