async templates working from S3

packages/builder/src/components/start/TemplateList.svelte

@@ -2,19 +2,13 @@
   import { Button, Heading } from "@budibase/bbui"
   import AppCard from "./AppCard.svelte"
   import Spinner from "components/common/Spinner.svelte"
+  import api from "builderStore/api"
 
   export let onSelect
 
-  let templates = []
+  async function fetchTemplates() {
-
+    const response = await api.get("/api/templates?type=app")
-  function fetchTemplates() {
+    return await response.json()
-    return Promise.resolve([
-      {
-        name: "Funky",
-        description: "Funky ass template",
-        minBuilderVersion: "",
-      },
-    ])
   }
 
   let templatesPromise = fetchTemplates()

packages/server/scripts/exportAppTemplate.js

@@ -1,36 +1,11 @@
 #!/usr/bin/env node
+const { exportTemplateFromApp } = require("../src/utilities/templates")
 
-// Script to export a budibase app into a package
+// Script to export a chosen budibase app into a package
-// Usage: foo
 
-const fs = require("fs-extra")
+const [name, instanceId, appId] = process.argv.slice(1)
-const path = require("path")
-const os = require("os")
-const replicationStream = require("pouchdb-replication-stream")
 
-const PouchDB = require("../src/db")
+exportTemplateFromApp({
-
-PouchDB.plugin(replicationStream.plugin)
-PouchDB.adapter("writableStream", replicationStream.adapters.writableStream)
-
-async function exportAppToTemplate({ instanceId, appId, templateName }) {
-  // Copy frontend files
-  console.log("Copying frontend definition...")
-  const appToExport = path.join(os.homedir(), ".budibase", appId, "pages")
-  const templateOutputPath = path.join(os.homedir(), ".budibase", templateName)
-  fs.copySync(appToExport, `${templateOutputPath}/pages`)
-
-  const writeStream = fs.createWriteStream(`${templateOutputPath}/dump.txt`)
-
-  // perform couch dump
-  const instanceDb = new PouchDB(instanceId)
-
-  console.log("Performing database dump..")
-  await instanceDb.dump(writeStream)
-  console.log("Export complete!")
-}
-
-exportAppToTemplate({
   templateName: "Funky",
   instanceId: "inst_b70abba_16feb394866140a1ac3f2e450e99f28a",
   appId: "b70abba3874546bf99a339911b579937",

packages/server/src/api/controllers/application.js

@@ -82,11 +82,10 @@ exports.create = async function(ctx) {
       },
     },
   }
-  // TODO: pass template into here from create InstCtx
+
   await instanceController.create(createInstCtx)
   newApplication.instances.push(createInstCtx.body)
 
-  // TODO: if template is passed, create the app package from the template and seed the instance database
   if (process.env.NODE_ENV !== "jest") {
     const newAppFolder = await createEmptyAppPackage(ctx, newApplication)
     await downloadExtractComponentLibraries(newAppFolder)
@@ -160,10 +159,16 @@ const createEmptyAppPackage = async (ctx, app) => {
     name: npmFriendlyAppName(app.name),
   })
 
-  // Copy the frontend page definition files from the template directory
+  // if this app is being created from a template,
-  // if this app is being created from a template.
+  // copy the frontend page definition files from
+  // the template directory.
   if (app.template) {
-    const templatePageDefinitions = join(appsFolder, app.template.name, "pages")
+    const templatePageDefinitions = join(
+      appsFolder,
+      "templates",
+      app.template.key,
+      "pages"
+    )
     await copy(templatePageDefinitions, join(appsFolder, app._id, "pages"))
   }
 

packages/server/src/api/controllers/instance.js

@@ -3,6 +3,7 @@ const CouchDB = require("../../db")
 const client = require("../../db/clientDb")
 const newid = require("../../db/newid")
 const { budibaseAppsDir } = require("../../utilities/budibaseDir")
+const { downloadTemplate } = require("../../utilities/templates")
 
 exports.create = async function(ctx) {
   const instanceName = ctx.request.body.name
@@ -54,13 +55,10 @@ exports.create = async function(ctx) {
   budibaseApp.instances.push(instance)
   await clientDb.put(budibaseApp)
 
-  // TODO: download the chosen template tar file from s3 and unpack it
   // replicate the template data to the instance DB
-  // TODO: templates should be downloaded to .budibase/templates/something
   if (template) {
-    const dbDumpReadStream = fs.createReadStream(
+    const templatePath = await downloadTemplate(...template.key.split("/"))
-      `${budibaseAppsDir()}/${template.name}/dump.txt`
+    const dbDumpReadStream = fs.createReadStream(`${templatePath}/db/dump.txt`)
-    )
     const { ok } = await db.load(dbDumpReadStream)
     if (!ok) {
       ctx.throw(500, "Error loading database dump from template.")

packages/server/src/api/controllers/static.js

@@ -2,7 +2,7 @@ const send = require("koa-send")
 const { resolve, join } = require("path")
 const jwt = require("jsonwebtoken")
 const fetch = require("node-fetch")
-const fs = require("fs")
+const fs = require("fs-extra")
 const uuid = require("uuid")
 const AWS = require("aws-sdk")
 const { prepareUploadForS3 } = require("./deploy/aws")
@@ -138,8 +138,6 @@ exports.performLocalFileProcessing = async function(ctx) {
 exports.serveApp = async function(ctx) {
   const mainOrAuth = ctx.isAuthenticated ? "main" : "unauthenticated"
 
-  console.log(ctx.user)
-
   // default to homedir
   const appPath = resolve(
     budibaseAppsDir(),

packages/server/src/api/controllers/templates.js

@@ -0,0 +1,44 @@
+const fetch = require("node-fetch")
+const {
+  downloadTemplate,
+  exportTemplateFromApp,
+} = require("../../utilities/templates")
+
+const DEFAULT_TEMPLATES_BUCKET =
+  "prod-budi-templates.s3-eu-west-1.amazonaws.com"
+
+exports.fetch = async function(ctx) {
+  const { type = "app" } = ctx.query
+
+  const response = await fetch(
+    `https://${DEFAULT_TEMPLATES_BUCKET}/manifest.json`
+  )
+  const json = await response.json()
+  ctx.body = Object.values(json.templates[type])
+}
+
+exports.downloadTemplate = async function(ctx) {
+  const { type, name } = ctx.params
+
+  await downloadTemplate(type, name)
+
+  ctx.body = {
+    message: `template ${type}:${name} downloaded successfully.`,
+  }
+}
+
+exports.exportTemplateFromApp = async function(ctx) {
+  const { appId, instanceId } = ctx.user.appId
+  const { templateName } = ctx.request.body
+
+  await exportTemplateFromApp({
+    appId,
+    instanceId,
+    templateName,
+  })
+
+  ctx.status = 200
+  ctx.body = {
+    message: `Created template: ${templateName}`,
+  }
+}

packages/server/src/api/index.js

@@ -19,6 +19,7 @@ const {
   automationRoutes,
   accesslevelRoutes,
   apiKeysRoutes,
+  templatesRoutes,
 } = require("./routes")
 
 const router = new Router()
@@ -89,6 +90,9 @@ router.use(automationRoutes.allowedMethods())
 
 router.use(deployRoutes.routes())
 router.use(deployRoutes.allowedMethods())
+
+router.use(templatesRoutes.routes())
+router.use(templatesRoutes.allowedMethods())
 // end auth routes
 
 router.use(pageRoutes.routes())

packages/server/src/api/routes/index.js

@@ -13,6 +13,7 @@ const automationRoutes = require("./automation")
 const accesslevelRoutes = require("./accesslevel")
 const deployRoutes = require("./deploy")
 const apiKeysRoutes = require("./apikeys")
+const templatesRoutes = require("./templates")
 
 module.exports = {
   deployRoutes,
@@ -30,4 +31,5 @@ module.exports = {
   automationRoutes,
   accesslevelRoutes,
   apiKeysRoutes,
+  templatesRoutes,
 }

packages/server/src/api/routes/templates.js

@@ -0,0 +1,17 @@
+const Router = require("@koa/router")
+const controller = require("../controllers/templates")
+const authorized = require("../../middleware/authorized")
+const { BUILDER } = require("../../utilities/accessLevels")
+
+const router = Router()
+
+router
+  .get("/api/templates", authorized(BUILDER), controller.fetch)
+  .get(
+    "/api/templates/:type/:name",
+    // authorized(BUILDER),
+    controller.downloadTemplate
+  )
+  .post("/api/templates", authorized(BUILDER), controller.exportTemplateFromApp)
+
+module.exports = router

packages/server/src/utilities/templates.js

@@ -0,0 +1,60 @@
+const path = require("path")
+const fs = require("fs-extra")
+const os = require("os")
+const fetch = require("node-fetch")
+const stream = require("stream")
+const tar = require("tar-fs")
+const zlib = require("zlib")
+const { promisify } = require("util")
+const streamPipeline = promisify(stream.pipeline)
+const { budibaseAppsDir } = require("./budibaseDir")
+const CouchDB = require("../db")
+
+const DEFAULT_TEMPLATES_BUCKET =
+  "prod-budi-templates.s3-eu-west-1.amazonaws.com"
+
+exports.downloadTemplate = async function(type, name) {
+  const templateUrl = `https://${DEFAULT_TEMPLATES_BUCKET}/templates/${type}/${name}.tar.gz`
+  console.log(templateUrl, type, name)
+  const response = await fetch(templateUrl)
+
+  if (!response.ok) {
+    throw new Error(
+      `Error downloading template ${type}:${name}: ${response.statusText}`
+    )
+  }
+
+  // stream the response, unzip and extract
+  await streamPipeline(
+    response.body,
+    zlib.Unzip(),
+    tar.extract(path.join(budibaseAppsDir(), "templates", type))
+  )
+
+  return path.join(budibaseAppsDir(), "templates", type, name)
+}
+
+exports.exportTemplateFromApp = async function({
+  appId,
+  templateName,
+  instanceId,
+}) {
+  // Copy frontend files
+  console.log("Copying frontend definition...")
+  const appToExport = path.join(os.homedir(), ".budibase", appId, "pages")
+  const templatesDir = path.join(os.homedir(), ".budibase", "templates")
+  fs.ensureDirSync(templatesDir)
+
+  const templateOutputPath = path.join(templatesDir, templateName)
+  fs.copySync(appToExport, `${templateOutputPath}/pages`)
+
+  fs.ensureDirSync(path.join(templateOutputPath, "db"))
+  const writeStream = fs.createWriteStream(`${templateOutputPath}/db/dump.txt`)
+
+  // perform couch dump
+  const instanceDb = new CouchDB(instanceId)
+
+  console.log("Performing database dump..")
+  await instanceDb.dump(writeStream)
+  console.log("Export complete!")
+}