s3 awareness, authentication through API keys

packages/builder/src/pages/[application]/deploy/index.svelte

@@ -1 +1,60 @@
-<img src="/_builder/assets/deploy_rocket.jpg" />
+<script>
+  import { Button } from "@budibase/bbui"
+  import { notifier } from "builderStore/store/notifications"
+  import api from "builderStore/api"
+
+  async function deployApp() {
+    const DEPLOY_URL = `/deploy`
+
+    try {
+      notifier.info("Starting Deployment..")
+      const response = await api.post(DEPLOY_URL)
+      const json = await response.json()
+      if (response.status !== 200) {
+        throw new Error
+      }
+
+      notifier.success("Deployment Complete. View your app at blah URL")
+    } catch (err) {
+      notifier.danger("Deployment unsuccessful. Please try again later.")
+    }
+  }
+</script>
+
+<section>
+  <div>
+    <h4>It's time to shine!</h4>
+    <Button secondary medium on:click={deployApp}>
+      Deploy App
+    </Button>
+  </div>
+  <img src="/_builder/assets/deploy-rocket.jpg" />
+</section>
+
+<style>
+  h4 {
+    color: var(--white);
+    font-size: 18px;
+    font-weight: bold;
+    margin-bottom: 30px;
+  }
+
+  section {
+    position: relative;
+  }
+
+  div {
+    position: absolute;
+    display: flex;
+    text-align: center;
+    flex-direction: column;
+    align-items: center;
+    justify-content: center;
+    left: 0; 
+    right: 0; 
+    top: 20%;
+    margin-left: auto; 
+    margin-right: auto; 
+    width: 50%;
+  }
+</style>

packages/server/deploy.sh

@@ -1,3 +1,3 @@
-docker build -t prod-apps-budi-live .
+docker build -t prod-budi-app-server .
-docker tag prod-apps-budi-live:latest 545012296077.dkr.ecr.eu-west-1.amazonaws.com/prod-apps-budi-live:latest
+docker tag prod-budi-app-server:latest 545012296077.dkr.ecr.eu-west-1.amazonaws.com/prod-budi-app-server:latest
-docker push 545012296077.dkr.ecr.eu-west-1.amazonaws.com/prod-apps-budi-live:latest 
+docker push 545012296077.dkr.ecr.eu-west-1.amazonaws.com/prod-budi-app-server:latest 

packages/server/src/api/controllers/deploy.js

@@ -1,83 +0,0 @@
-const fs = require("fs")
-const AWS = require("aws-sdk")
-const CouchDB = require("../../db")
-const {
-  budibaseAppsDir,
-} = require("../../utilities/budibaseDir")
-
-const s3 = new AWS.S3({
-  params: {
-    Bucket: process.env.BUDIBASE_APP_ASSETS_BUCKET
-  }
-})
-
-async function uploadAppAssets({ clientId, appId }) {
-  const appAssetsPath = `${budibaseAppsDir()}/${appId}/public`
-
-  const appPages = fs.readdirSync(appAssetsPath)
-
-  const uploads = []
-
-  for (let page of appPages) {
-    for (let filename of fs.readdirSync(`${appAssetsPath}/${page}`)) {
-      const filePath =  `${appAssetsPath}/${page}/${filename}`
-      const stat = await fs.lstatSync(filePath)
-      
-      // TODO: need to account for recursively traversing dirs
-      if (stat.isFile()) {
-        const fileBytes = fs.readFileSync(`${appAssetsPath}/${page}/${filename}`)
-
-        const upload = s3.upload({
-          Key: `${clientId}/${appId}/${page}/${filename}`,
-          Body: fileBytes
-        }).promise()
-
-        uploads.push(upload)
-      }
-    }
-  }
-
-  try {
-    await Promise.all(uploads)
-  } catch (err) {
-    console.error("Error uploading budibase app assets to s3", err)
-    throw err
-  }
-}
-
-function replicateCouch(instanceId) {
-  return new Promise((resolve, reject) => {
-    const localDb = new CouchDB(instanceId)
-    const remoteDb = new CouchDB(`${process.env.COUCH_DB_REMOTE}/${instanceId}`)
-
-    const replication = localDb.replicate.to(remoteDb);
-
-    replication.on("complete", () => resolve())
-    replication.on("error", err => reject(err))
-  });
-
-}
-
-exports.deployApp = async function(ctx) {
-  const { 
-    user = {
-      instanceId: "test_instance_id",
-      clientId: "test123", 
-      appId: "24602c068a2d494cb09cb48f44bfdd8f"
-    } 
-  } = ctx;
-
-  // TODO: This should probably be async - it could take a while  
-  try {
-    // upload all the assets to s3
-    await uploadAppAssets(user)
-
-    // replicate the DB to the couchDB cluster in prod
-    await replicateCouch(user.instanceId);
-
-    ctx.body = "Deployment Successful!"
-
-    } catch (err) {
-      ctx.throw(err.status || 500, `Deployment Failed: ${err.message}`);
-    }
-}

packages/server/src/api/controllers/deploy/aws.js

@@ -0,0 +1,79 @@
+const fs = require("fs")
+const AWS = require("aws-sdk")
+const fetch = require("node-fetch")
+const {
+  budibaseAppsDir,
+} = require("../../../utilities/budibaseDir")
+
+async function fetchTemporaryCredentials() {
+  const CREDENTIALS_URL = "https://dt4mpwwap8.execute-api.eu-west-1.amazonaws.com/prod/"
+
+  const response = await fetch(CREDENTIALS_URL, {
+    method: "POST",
+    body: JSON.stringify({
+      apiKey: "d498278c-4ab4-144b-c212-b8f9e6da5c2b"
+    })
+  })
+
+  if (response.status !== 200) {
+    throw new Error
+  }
+
+  const json = await response.json()
+
+  return json
+} 
+
+exports.uploadAppAssets = async function ({ appId }) {
+  const { credentials, accountId } = await fetchTemporaryCredentials()
+  console.log({
+    credentials,
+    accountId
+  });
+
+  AWS.config.update({
+    accessKeyId: credentials.AccessKeyId, 
+    secretAccessKey: credentials.SecretAccessKey,
+    sessionToken: credentials.SessionToken
+  });
+
+  const s3 = new AWS.S3({
+    params: {
+      Bucket: process.env.BUDIBASE_APP_ASSETS_BUCKET
+    }
+  })
+
+  const appAssetsPath = `${budibaseAppsDir()}/${appId}/public`
+
+  const appPages = fs.readdirSync(appAssetsPath)
+
+  const uploads = []
+
+  for (let page of appPages) {
+    for (let filename of fs.readdirSync(`${appAssetsPath}/${page}`)) {
+      const filePath =  `${appAssetsPath}/${page}/${filename}`
+      const stat = await fs.lstatSync(filePath)
+      
+      // TODO: need to account for recursively traversing dirs
+      if (stat.isFile()) {
+        const fileBytes = fs.readFileSync(`${appAssetsPath}/${page}/${filename}`)
+
+        console.log(`${accountId}/${appId}/${page}/${filename}`)
+
+        const upload = s3.upload({
+          Key: `assets/${accountId}/${appId}/${page}/${filename}`,
+          Body: fileBytes
+        }).promise()
+
+        uploads.push(upload)
+      }
+    }
+  }
+
+  try {
+    await Promise.all(uploads)
+  } catch (err) {
+    console.error("Error uploading budibase app assets to s3", err)
+    throw err
+  }
+}

packages/server/src/api/controllers/deploy/index.js

@@ -0,0 +1,44 @@
+const CouchDB = require("pouchdb")
+const PouchDB = require("../../../db")
+const { 
+  uploadAppAssets,
+} = require("./aws")
+
+function replicateCouch(instanceId) {
+  return new Promise((resolve, reject) => {
+    const localDb = new PouchDB(instanceId)
+    const remoteDb = new CouchDB(`${process.env.COUCH_DB_REMOTE}/${instanceId}`)
+
+    const replication = localDb.replicate.to(remoteDb);
+
+    replication.on("complete", () => resolve())
+    replication.on("error", err => reject(err))
+  });
+}
+
+exports.deployApp = async function(ctx) {
+  // TODO: This should probably be async - it could take a while  
+  try {
+    const clientAppLookupDB = new PouchDB("client_app_lookup")
+    console.log(ctx.user)
+    const { clientId } = await clientAppLookupDB.get(ctx.user.appId)
+
+    ctx.log.info(`Uploading assets for appID ${ctx.user.appId} assets to s3..`);
+    await uploadAppAssets({ 
+      clientId,
+      appId: ctx.user.appId
+    })
+
+    // replicate the DB to the couchDB cluster in prod
+    ctx.log.info("Replicating local PouchDB to remote..");
+    await replicateCouch(ctx.user.instanceId);
+
+    ctx.body = { 
+      status: "SUCCESS", 
+      completed: Date.now()
+    }
+
+    } catch (err) {
+      ctx.throw(err.status || 500, `Deployment Failed: ${err.message}`);
+    }
+}

packages/server/src/api/controllers/static.js

@@ -8,8 +8,7 @@ const setBuilderToken = require("../../utilities/builder/setBuilderToken")
 const { ANON_LEVEL_ID } = require("../../utilities/accessLevels")
 const jwt = require("jsonwebtoken")
 const fetch = require("node-fetch")
-
+const { S3 } = require("aws-sdk")
-const S3_URL_PREFIX = "https://s3-eu-west-1.amazonaws.com/apps.budibase.com"
 
 exports.serveBuilder = async function(ctx) {
   let builderPath = resolve(__dirname, "../../../builder")
@@ -47,15 +46,16 @@ exports.serveApp = async function(ctx) {
 
   const { file = "index.html" } = ctx
 
+
   if (ctx.isCloud) {
-    const requestUrl = `${S3_URL_PREFIX}/${ctx.params.appId}/public/${mainOrAuth}/${file}`
+    const S3_URL = `https://${ctx.params.appId}.app.budi.live/assets/${ctx.params.appId}/public/${mainOrAuth}/${file}`
-    console.log('request url:' , requestUrl)
+    const response = await fetch(S3_URL)
-    const response = await fetch(requestUrl)
     const body = await response.text()
     ctx.body = body 
-  } else {
+    return
-    await send(ctx, file, { root: ctx.devPath || appPath })
   }
+
+  await send(ctx, file, { root: ctx.devPath || appPath })
 }
 
 exports.serveAppAsset = async function(ctx) {
@@ -69,15 +69,15 @@ exports.serveAppAsset = async function(ctx) {
     mainOrAuth
   )
 
-  if (ctx.isCloud) {
+  // if (ctx.isCloud) {
-    const requestUrl = `${S3_URL_PREFIX}/${appId}/public/${mainOrAuth}/${ctx.file || "index.html"}`
+  //   const requestUrl = `${S3_URL_PREFIX}/${appId}/public/${mainOrAuth}/${ctx.file || "index.html"}`
-    console.log('request url:' , requestUrl)
+  //   console.log('request url:' , requestUrl)
-    const response = await fetch(requestUrl)
+  //   const response = await fetch(requestUrl)
-    const body = await response.text()
+  //   const body = await response.text()
-    ctx.body = body 
+  //   ctx.body = body 
-  } else {
+  // } else {
     await send(ctx, ctx.file, { root: ctx.devPath || appPath })
-  }
+  // }
 }
 
 exports.serveComponentLibrary = async function(ctx) {
@@ -98,16 +98,16 @@ exports.serveComponentLibrary = async function(ctx) {
     )
   }
 
-  if (ctx.isCloud) {
+  // if (ctx.isCloud) {
-    const appId = ctx.user.appId
+  //   const appId = ctx.user.appId
-    const requestUrl = encodeURI(`${S3_URL_PREFIX}/${appId}/node_modules/${ctx.query.library}/dist/index.js`)
+  //   const requestUrl = encodeURI(`${S3_URL_PREFIX}/${appId}/node_modules/${ctx.query.library}/dist/index.js`)
-    console.log('request url components: ', requestUrl)
+  //   console.log('request url components: ', requestUrl)
-    const response = await fetch(requestUrl)
+  //   const response = await fetch(requestUrl)
-    const body = await response.text()
+  //   const body = await response.text()
-    ctx.type = 'application/javascript'
+  //   ctx.type = 'application/javascript'
-    ctx.body = body;
+  //   ctx.body = body;
-    return
+  //   return
-  }
+  // }
 
   await send(ctx, "/index.js", { root: componentLibraryPath })
 }

packages/server/src/api/index.js

@@ -42,8 +42,8 @@ router
       jwtSecret: env.JWT_SECRET,
       useAppRootPath: true,
     }
-    // ctx.isDev = env.NODE_ENV !== "production" && env.NODE_ENV !== "jest"
+    ctx.isDev = env.NODE_ENV !== "production" && env.NODE_ENV !== "jest"
-    ctx.isCloud = true
+    // ctx.isCloud = true
     await next()
   })
   .use(authenticated)

packages/server/src/api/routes/deploy.js

@@ -6,6 +6,6 @@ const { BUILDER } = require("../../utilities/accessLevels")
 const router = Router()
 
 // router.post("/:appId/deploy", authorized(BUILDER), controller.deployApp)
-router.post("/:appId/deploy", controller.deployApp)
+router.post("/deploy", controller.deployApp)
 
 module.exports = router

packages/server/src/app.js

@@ -16,7 +16,7 @@ app.use(
     prettyPrint: {
       levelFirst: true,
     },
-    level: env.LOG_LEVEL || "error",
+    level: "info" || "info",
   })
 )