Adding a deployment service which takes over from the lambdas in local operation, this may become part of the hosting portal if we ever decide to opensource that part of it.

hosting/deployment

@@ -0,0 +1 @@
+../packages/deployment/

hosting/docker-compose.yml

@@ -16,13 +16,30 @@ services:
       BUDIBASE_ENVIRONMENT: ${BUDIBASE_ENVIRONMENT}
       LOGO_URL: ${LOGO_URL}
       PORT: ${APP_PORT}
+    depends_on:
+      - deployment-service
+
+  deployment-service:
+    build: ./deployment
+    ports:
+      - "${DEPLOYMENT_PORT}:${DEPLOYMENT_PORT}"
+    environment:
+      SELF_HOSTED: 1,
+      DEPLOYMENT_API_KEY: ${DEPLOYMENT_API_KEY}
+      PORT: ${DEPLOYMENT_PORT}
+      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
+      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
+      RAW_MINIO_URL: http://nginx-service:${MINIO_PORT}
+      COUCH_DB_USERNAME: ${COUCH_DB_USER}
+      COUCH_DB_PASSWORD: ${COUCH_DB_PASSWORD}
+      RAW_COUCH_DB_URL: http://couchdb-service:5984
     depends_on:
       - nginx-service
       - minio-service
       - couch-init
 
   minio-service:
-    image: minio/minio:RELEASE.2020-12-10T01-54-29Z
+    image: minio/minio
     volumes:
       - data1:/data
     ports:

hosting/hosting.properties

@@ -2,9 +2,11 @@ MINIO_ACCESS_KEY=budibase
 MINIO_SECRET_KEY=budibase
 COUCH_DB_PASSWORD=budibase
 COUCH_DB_USER=budibase
+DEPLOYMENT_API_KEY=budibase
 BUDIBASE_ENVIRONMENT=PRODUCTION
 HOSTING_URL="http://localhost:4001"
 LOGO_URL=https://logoipsum.com/logo/logo-15.svg
 APP_PORT=4002
 MINIO_PORT=4003
 COUCH_DB_PORT=4004
+DEPLOYMENT_PORT=4006

packages/builder/src/components/start/BuilderSettingsModal.svelte

@@ -40,7 +40,7 @@
   <Toggle thin text="Self hosted" bind:checked={selfhosted} />
   {#if selfhosted}
     <Input bind:value={hostingInfo.appServerUrl} label="Apps URL" />
-    <Input bind:value={hostingInfo.objectStoreUrl} label="Object store URL" />
+    <Input bind:value={hostingInfo.deploymentServerUrl} label="Deployments URL" />
     <Toggle thin text="HTTPS" bind:checked={hostingInfo.useHttps} />
   {/if}
 </ModalContent>

packages/deployment/.gitignore

@@ -0,0 +1,2 @@
+node_modules/
+.env

packages/deployment/Dockerfile

@@ -0,0 +1,15 @@
+FROM node:12-alpine
+
+WORKDIR /app
+
+# copy files and install dependencies
+COPY . ./
+RUN yarn
+
+EXPOSE 4001
+
+# have to add node environment production after install
+# due to this causing yarn to stop installing dev dependencies
+# which are actually needed to get this environment up and running
+ENV NODE_ENV=production
+CMD ["yarn", "run:docker"]

packages/deployment/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "@budibase/deployment",
+  "email": "hi@budibase.com",
+  "version": "0.3.8",
+  "description": "Budibase Deployment Server",
+  "main": "src/index.js",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/Budibase/budibase.git"
+  },
+  "keywords": [
+    "budibase"
+  ],
+  "scripts": {
+    "run:docker": "node src/index.js"
+  },
+  "author": "Budibase",
+  "license": "AGPL-3.0-or-later",
+  "dependencies": {
+    "@koa/router": "^8.0.0",
+    "aws-sdk": "^2.811.0",
+    "got": "^11.8.1",
+    "joi": "^17.2.1",
+    "koa": "^2.7.0",
+    "koa-body": "^4.2.0",
+    "koa-compress": "^4.0.1",
+    "koa-pino-logger": "^3.0.0",
+    "koa-send": "^5.0.0",
+    "koa-session": "^5.12.0",
+    "koa-static": "^5.0.0",
+    "pino-pretty": "^4.0.0",
+    "server-destroy": "^1.0.1"
+  }
+}

packages/deployment/src/api/controllers/deploy.js

@@ -0,0 +1,86 @@
+const env = require("../../environment")
+const got = require("got")
+const AWS = require("aws-sdk")
+
+const APP_BUCKET = "app-assets"
+// this doesn't matter in self host
+const REGION = "eu-west-1"
+
+async function getCouchSession() {
+  // fetch session token for the api user
+  const session = await got.post(`${env.RAW_COUCH_DB_URL}/_session`, {
+    responseType: "json",
+    json: {
+      username: env.COUCH_DB_USERNAME,
+      password: env.COUCH_DB_PASSWORD,
+    }
+  })
+
+  const cookie = session.headers["set-cookie"][0]
+  // Get the session cookie value only
+  return cookie.split(";")[0]
+}
+
+async function getMinioSession() {
+  AWS.config.update({
+    accessKeyId: env.MINIO_ACCESS_KEY,
+    secretAccessKey: env.MINIO_SECRET_KEY,
+  })
+
+  // make sure the bucket exists
+  const objClient = new AWS.S3({
+    endpoint: env.RAW_MINIO_URL,
+    region: REGION,
+    s3ForcePathStyle: true, // needed with minio?
+    params: {
+      Bucket: APP_BUCKET,
+    },
+  })
+  // make sure the bucket exists
+  try {
+    await objClient.headBucket({ Bucket: APP_BUCKET }).promise()
+  } catch (err) {
+    // bucket doesn't exist create it
+    if (err.statusCode === 404) {
+      await objClient.createBucket({ Bucket: APP_BUCKET }).promise()
+    } else {
+      throw err
+    }
+  }
+  // TODO: this doesn't seem to work get an error
+  // TODO: Generating temporary credentials not allowed for this request.
+  // TODO: this should work based on minio documentation
+  // const sts = new AWS.STS({
+  //   endpoint: env.RAW_MINIO_URL,
+  //   region: REGION,
+  //   s3ForcePathStyle: true,
+  // })
+  // // NOTE: In the following commands RoleArn and RoleSessionName are not meaningful for MinIO
+  // const params = {
+  //   DurationSeconds: 3600,
+  //   ExternalId: "123ABC",
+  //   Policy: '{"Version":"2012-10-17","Statement":[{"Sid":"Stmt1","Effect":"Allow","Action":"s3:*","Resource":"arn:aws:s3:::*"}]}',
+  //   RoleArn: 'arn:xxx:xxx:xxx:xxxx',
+  //   RoleSessionName: 'anything',
+  // };
+  // const assumedRole = await sts.assumeRole(params).promise();
+  // if (!assumedRole) {
+  //   throw "Unable to get access to object store."
+  // }
+  // return assumedRole.Credentials
+  // TODO: need to do something better than this
+  return {
+    accessKeyId: env.MINIO_ACCESS_KEY,
+    secretAccessKey: env.MINIO_SECRET_KEY,
+  }
+}
+
+exports.deploy = async ctx => {
+  ctx.body = {
+    couchDbSession: await getCouchSession(),
+    bucket: APP_BUCKET,
+    objectStoreSession: await getMinioSession(),
+    couchDbUrl: env.RAW_COUCH_DB_URL,
+    objectStoreUrl: env.RAW_MINIO_URL,
+  }
+}

packages/deployment/src/api/index.js

@@ -0,0 +1,45 @@
+const Router = require("@koa/router")
+const compress = require("koa-compress")
+const zlib = require("zlib")
+const { routes } = require("./routes")
+
+const router = new Router()
+
+router
+  .use(
+    compress({
+      threshold: 2048,
+      gzip: {
+        flush: zlib.Z_SYNC_FLUSH,
+      },
+      deflate: {
+        flush: zlib.Z_SYNC_FLUSH,
+      },
+      br: false,
+    })
+  )
+  .use("/health", ctx => (ctx.status = 200))
+
+// error handling middleware
+router.use(async (ctx, next) => {
+  try {
+    await next()
+  } catch (err) {
+    ctx.log.error(err)
+    ctx.status = err.status || err.statusCode || 500
+    ctx.body = {
+      message: err.message,
+      status: ctx.status,
+    }
+  }
+})
+
+router.get("/health", ctx => (ctx.status = 200))
+
+// authenticated routes
+for (let route of routes) {
+  router.use(route.routes())
+  router.use(route.allowedMethods())
+}
+
+module.exports = router

packages/deployment/src/api/routes/deploy.js

@@ -0,0 +1,10 @@
+const Router = require("@koa/router")
+const controller = require("../controllers/deploy")
+const checkKey = require("../../middleware/check-key")
+
+const router = Router()
+
+router
+  .post("/api/deploy", checkKey, controller.deploy)
+
+module.exports = router

packages/deployment/src/api/routes/index.js

@@ -0,0 +1,5 @@
+const deployRoutes = require("./deploy")
+
+exports.routes = [
+  deployRoutes,
+]

packages/deployment/src/environment.js

@@ -0,0 +1,15 @@
+module.exports = {
+  SELF_HOSTED: process.env.SELF_HOSTED,
+  DEPLOYMENT_API_KEY: process.env.DEPLOYMENT_API_KEY,
+  PORT: process.env.PORT,
+  MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
+  MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
+  RAW_MINIO_URL: process.env.RAW_MINIO_URL,
+  COUCH_DB_USERNAME: process.env.COUCH_DB_USERNAME,
+  COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,
+  RAW_COUCH_DB_URL: process.env.RAW_COUCH_DB_URL,
+  _set(key, value) {
+    process.env[key] = value
+    module.exports[key] = value
+  },
+}

packages/deployment/src/index.js

@@ -0,0 +1,48 @@
+const Koa = require("koa")
+const destroyable = require("server-destroy")
+const koaBody = require("koa-body")
+const logger = require("koa-pino-logger")
+const http = require("http")
+const api = require("./api")
+const env = require("./environment")
+
+const app = new Koa()
+
+if (!env.SELF_HOSTED) {
+  throw "Currently this service only supports use in self hosting"
+}
+
+// set up top level koa middleware
+app.use(koaBody({ multipart: true }))
+
+app.use(
+  logger({
+    prettyPrint: {
+      levelFirst: true,
+    },
+    level: env.LOG_LEVEL || "error",
+  })
+)
+
+// api routes
+app.use(api.routes())
+
+const server = http.createServer(app.callback())
+destroyable(server)
+
+server.on("close", () => console.log("Server Closed"))
+
+module.exports = server.listen(env.PORT || 4002, async () => {
+  console.log(`Deployment running on ${JSON.stringify(server.address())}`)
+})
+
+process.on("uncaughtException", err => {
+  console.error(err)
+  server.close()
+  server.destroy()
+})
+
+process.on("SIGTERM", () => {
+  server.close()
+  server.destroy()
+})

packages/deployment/src/middleware/check-key.js

@@ -0,0 +1,4 @@
+module.exports = async (ctx, next) => {
+  // TODO: need to check the API key provided in the header
+  await next()
+}

packages/server/src/api/controllers/deploy/awsDeploy.js

@@ -1,9 +1,11 @@
 const AWS = require("aws-sdk")
 const fetch = require("node-fetch")
 const env = require("../../../environment")
-const { deployToObjectStore, performReplication } = require("./utils")
+const {
-const CouchDB = require("pouchdb")
+  deployToObjectStore,
-const PouchDB = require("../../../db")
+  performReplication,
+  fetchCredentials,
+} = require("./utils")
 
 /**
  * Verifies the users API key and
@@ -12,26 +14,12 @@ const PouchDB = require("../../../db")
  * @param {object} deployment - information about the active deployment, including the appId and quota.
  */
 exports.preDeployment = async function(deployment) {
-  const response = await fetch(env.DEPLOYMENT_CREDENTIALS_URL, {
+  const json = await fetchCredentials(env.DEPLOYMENT_CREDENTIALS_URL, {
-    method: "POST",
+    apiKey: env.BUDIBASE_API_KEY,
-    body: JSON.stringify({
+    appId: deployment.getAppId(),
-      apiKey: env.BUDIBASE_API_KEY,
+    quota: deployment.getQuota(),
-      appId: deployment.getAppId(),
-      quota: deployment.getQuota(),
-    }),
   })
 
-  const json = await response.json()
-  if (json.errors) {
-    throw new Error(json.errors)
-  }
-
-  if (response.status !== 200) {
-    throw new Error(
-      `Error fetching temporary credentials for api key: ${env.BUDIBASE_API_KEY}`
-    )
-  }
-
   // set credentials here, means any time we're verified we're ready to go
   if (json.credentials) {
     AWS.config.update({
@@ -88,14 +76,10 @@ exports.deploy = async function(deployment) {
 
 exports.replicateDb = async function(deployment) {
   const appId = deployment.getAppId()
-  const { session } = deployment.getVerification()
+  const verification = deployment.getVerification()
-  const localDb = new PouchDB(appId)
+  return performReplication(
-  const remoteDb = new CouchDB(`${env.DEPLOYMENT_DB_URL}/${appId}`, {
+    appId,
-    fetch: function(url, opts) {
+    verification.couchDbSession,
-      opts.headers.set("Cookie", `${session};`)
+    env.DEPLOYMENT_DB_URL
-      return CouchDB.fetch(url, opts)
+  )
-    },
-  })
-
-  return performReplication(localDb, remoteDb)
 }

packages/server/src/api/controllers/deploy/selfDeploy.js

@@ -1,16 +1,31 @@
 const env = require("../../../environment")
 const AWS = require("aws-sdk")
-const { deployToObjectStore, performReplication } = require("./utils")
+const {
-const CouchDB = require("pouchdb")
+  deployToObjectStore,
-const PouchDB = require("../../../db")
+  performReplication,
-
+  fetchCredentials,
-const APP_BUCKET = "app-assets"
+} = require("./utils")
+const { getDeploymentUrl } = require("../../../utilities/builder/hosting")
+const { join } = require("path")
 
 exports.preDeployment = async function() {
-  AWS.config.update({
+  const url = join(await getDeploymentUrl(), "api", "deploy")
-    accessKeyId: env.MINIO_ACCESS_KEY,
+  const json = await fetchCredentials(url, {
-    secretAccessKey: env.MINIO_SECRET_KEY,
+    apiKey: env.BUDIBASE_API_KEY,
   })
+
+  // response contains:
+  // couchDbSession, bucket, objectStoreSession, couchDbUrl, objectStoreUrl
+
+  // set credentials here, means any time we're verified we're ready to go
+  if (json.objectStoreSession) {
+    AWS.config.update({
+      accessKeyId: json.objectStoreSession.AccessKeyId,
+      secretAccessKey: json.objectStoreSession.SecretAccessKey,
+    })
+  }
+
+  return json
 }
 
 exports.postDeployment = async function() {
@@ -19,25 +34,15 @@ exports.postDeployment = async function() {
 
 exports.deploy = async function(deployment) {
   const appId = deployment.getAppId()
-  var objClient = new AWS.S3({
+  const verification = deployment.getVerification()
-    endpoint: env.MINIO_URL,
+  const objClient = new AWS.S3({
+    endpoint: verification.objectStoreUrl,
     s3ForcePathStyle: true, // needed with minio?
     signatureVersion: "v4",
     params: {
-      Bucket: APP_BUCKET,
+      Bucket: verification.bucket,
     },
   })
-  // checking the bucket exists
-  try {
-    await objClient.headBucket({ Bucket: APP_BUCKET }).promise()
-  } catch (err) {
-    // bucket doesn't exist create it
-    if (err.statusCode === 404) {
-      await objClient.createBucket({ Bucket: APP_BUCKET }).promise()
-    } else {
-      throw err
-    }
-  }
   // no metadata, aws has account ID in metadata
   const metadata = {}
   await deployToObjectStore(appId, objClient, metadata)
@@ -45,8 +50,10 @@ exports.deploy = async function(deployment) {
 
 exports.replicateDb = async function(deployment) {
   const appId = deployment.getAppId()
-  const localDb = new PouchDB(appId)
+  const verification = deployment.getVerification()
-
+  return performReplication(
-  const remoteDb = new CouchDB(`${env.COUCH_DB_URL}/${appId}`)
+    appId,
-  return performReplication(localDb, remoteDb)
+    verification.couchDbSession,
+    verification.couchDbUrl
+  )
 }

packages/server/src/api/controllers/deploy/utils.js

@@ -4,6 +4,7 @@ const { walkDir } = require("../../../utilities")
 const { join } = require("../../../utilities/centralPath")
 const { budibaseAppsDir } = require("../../../utilities/budibaseDir")
 const PouchDB = require("../../../db")
+const CouchDB = require("pouchdb")
 
 const CONTENT_TYPE_MAP = {
   html: "text/html",
@@ -11,6 +12,26 @@ const CONTENT_TYPE_MAP = {
   js: "application/javascript",
 }
 
+exports.fetchCredentials = async function(url, body) {
+  const response = await fetch(url, {
+    method: "POST",
+    body: JSON.stringify(body),
+  })
+
+  const json = await response.json()
+  if (json.errors) {
+    throw new Error(json.errors)
+  }
+
+  if (response.status !== 200) {
+    throw new Error(
+      `Error fetching temporary credentials for api key: ${body.apiKey}`
+    )
+  }
+
+  return json
+}
+
 exports.prepareUpload = async function({ s3Key, metadata, client, file }) {
   const extension = [...file.name.split(".")].pop()
   const fileBytes = fs.readFileSync(file.path)
@@ -89,8 +110,17 @@ exports.deployToObjectStore = async function(appId, objectClient, metadata) {
   }
 }
 
-exports.performReplication = (local, remote) => {
+exports.performReplication = (appId, session, dbUrl) => {
   return new Promise((resolve, reject) => {
+    const local = new PouchDB(appId)
+
+    const remote = new CouchDB(`${dbUrl}/${appId}`, {
+      fetch: function(url, opts) {
+        opts.headers.set("Cookie", `${session};`)
+        return CouchDB.fetch(url, opts)
+      },
+    })
+
     const replication = local.sync(remote)
 
     replication.on("complete", () => resolve())

packages/server/src/utilities/builder/hosting.js

@@ -24,7 +24,7 @@ exports.getHostingInfo = async () => {
       _id: HOSTING_DOC,
       type: exports.HostingTypes.CLOUD,
       appServerUrl: "app.budi.live",
-      objectStoreUrl: "cdn.app.budi.live",
+      deploymentServerUrl: "",
       templatesUrl: "prod-budi-templates.s3-eu-west-1.amazonaws.com",
       useHttps: true,
     }
@@ -44,6 +44,12 @@ exports.getAppServerUrl = async appId => {
   return url
 }
 
+exports.getDeploymentUrl = async () => {
+  const hostingInfo = await exports.getHostingInfo()
+  const protocol = getProtocol(hostingInfo)
+  return `${protocol}${hostingInfo.deploymentServerUrl}`
+}
+
 exports.getTemplatesUrl = async (appId, type, name) => {
   const hostingInfo = await exports.getHostingInfo()
   const protocol = getProtocol(hostingInfo)