Updating templates to be able to run locally with an environment variable LOCAL_TEMPLATES and making them work using the DB. Users are also no longer included in the db dump.

packages/builder/src/builderStore/store/frontend.js

@@ -1,23 +1,15 @@
-import { writable, get } from "svelte/store"
-import { cloneDeep } from "lodash/fp"
-import {
-  createProps,
-  makePropsSafe,
-  getBuiltin,
-} from "components/userInterface/pagesParsing/createProps"
+import { get, writable } from "svelte/store"
+import {cloneDeep} from "lodash/fp"
+import { createProps, getBuiltin, makePropsSafe } from "components/userInterface/pagesParsing/createProps"
 import { getExactComponent } from "components/userInterface/pagesParsing/searchComponents"
-import { backendUiStore, allScreens } from "builderStore"
+import {allScreens, backendUiStore } from "builderStore"
 import { generate_screen_css } from "../generate_css"
 import { fetchComponentLibDefinitions } from "../loadComponentLibraries"
 import api from "../api"
 import { DEFAULT_PAGES_OBJECT } from "../../constants"
 import getNewComponentName from "../getNewComponentName"
 import analytics from "analytics"
-import {
-  getParent,
-  generateNewIdsForComponent,
-  getComponentDefinition, findChildComponentType,
-} from "../storeUtils"
+import { findChildComponentType, generateNewIdsForComponent, getComponentDefinition, getParent } from "../storeUtils"
 
 const INITIAL_FRONTEND_STATE = {
   apps: [],
@@ -332,8 +324,7 @@ export const getFrontendStore = () => {
               ...presetProps,
               _instanceId: instanceId,
               _instanceName: instanceName,
-            },
-            state
+            }
           )
 
           const currentComponent =
@@ -367,8 +358,7 @@ export const getFrontendStore = () => {
       },
       copy: (component, cut = false) => {
         store.update(state => {
-          const copiedComponent = cloneDeep(component)
-          state.componentToPaste = copiedComponent
+          state.componentToPaste = cloneDeep(component)
           state.componentToPaste.isCut = cut
           if (cut) {
             const parent = getParent(
@@ -469,9 +459,7 @@ export const getFrontendStore = () => {
         const IdList = allComponents.map(c => c._id)
 
         // Construct ID Path:
-        const path = IdList.join("/")
-
-        return path
+        return IdList.join("/")
       },
       links: {
         save: async (url, title) => {

packages/builder/src/pages/index.svelte

@@ -62,7 +62,7 @@
     </div>
   </div>
 
-  <!-- <TemplateList onSelect={selectTemplate} /> -->
+  <TemplateList onSelect={selectTemplate} />
 
   <AppList />
 </div>

packages/builder/tests/buildCodeForScreen.spec.js

@@ -1,63 +0,0 @@
-import { buildCodeForScreens } from "../src/builderStore/buildCodeForScreens"
-
-describe("buildCodeForScreen", () => {
-  it("should package _code into runnable function, for simple screen props", () => {
-    const screen = {
-      props: {
-        _id: "1234",
-        _code: "render('render argument');",
-      },
-    }
-
-    let renderArg
-    const render = arg => {
-      renderArg = arg
-    }
-    const uiFunctions = getFunctions(screen)
-
-    const targetfunction = uiFunctions[screen.props._id]
-    expect(targetfunction).toBeDefined()
-
-    targetfunction(render)
-
-    expect(renderArg).toBe("render argument")
-  })
-
-  it("should package _code into runnable function, for _children ", () => {
-    const screen = {
-      props: {
-        _id: "parent",
-        _code: "render('parent argument');",
-        _children: [
-          {
-            _id: "child1",
-            _code: "render('child 1 argument');",
-          },
-          {
-            _id: "child2",
-            _code: "render('child 2 argument');",
-          },
-        ],
-      },
-    }
-
-    let renderArg
-    const render = arg => {
-      renderArg = arg
-    }
-    const uiFunctions = getFunctions(screen)
-
-    const targetfunction = uiFunctions["child2"]
-    expect(targetfunction).toBeDefined()
-
-    targetfunction(render)
-
-    expect(renderArg).toBe("child 2 argument")
-  })
-})
-
-const getFunctions = screen => {
-  const code = buildCodeForScreens([screen])
-  const func = new Function(`return ${code}`)()
-  return func
-}

packages/server/scripts/exportAppTemplate.js

@@ -23,6 +23,12 @@ yargs
     },
     async args => {
       console.log("Exporting app..")
+      if (args.name == null || args.appId == null) {
+        console.error(
+          "Unable to export without a name and app ID being specified, check help for more info."
+        )
+        return
+      }
       const exportPath = await exportTemplateFromApp({
         templateName: args.name,
         appId: args.appId,

packages/server/src/api/controllers/application.js

@@ -1,7 +1,7 @@
 const CouchDB = require("../../db")
 const { compileStaticAssetsForPage } = require("../../utilities/builder")
 const env = require("../../environment")
-const { copy, existsSync } = require("fs-extra")
+const { existsSync } = require("fs-extra")
 const { budibaseAppsDir } = require("../../utilities/budibaseDir")
 const setBuilderToken = require("../../utilities/builder/setBuilderToken")
 const fs = require("fs-extra")
@@ -160,21 +160,6 @@ const createEmptyAppPackage = async (ctx, app) => {
   }
 
   fs.mkdirpSync(newAppFolder)
-  // if this app is being created from a template,
-  // copy the frontend page definition files from
-  // the template directory.
-  if (app.template) {
-    const templatePageDefinitions = join(
-      appsFolder,
-      "templates",
-      app.template.key,
-      "pages"
-    )
-    // TODO: write the template page JSON to couch
-    // iterate over the pages and write them to the db
-    await copy(templatePageDefinitions, join(appsFolder, app._id, "pages"))
-  }
-
   const mainPage = cloneDeep(MAIN)
   mainPage._id = generatePageID()
   mainPage.title = app.name

packages/server/src/api/controllers/page.js

@@ -8,11 +8,7 @@ exports.save = async function(ctx) {
   const appPackage = ctx.request.body
 
   const page = await db.get(ctx.params.pageId)
-  await compileStaticAssetsForPage(
-    ctx.user.appId,
-    page.name,
-    ctx.request.body
-  )
+  await compileStaticAssetsForPage(ctx.user.appId, page.name, ctx.request.body)
 
   // remove special doc props which couch will complain about
   delete appPackage.page._css

packages/server/src/api/controllers/templates.js

@@ -2,25 +2,34 @@ const fetch = require("node-fetch")
 const {
   downloadTemplate,
   exportTemplateFromApp,
+  getLocalTemplates,
 } = require("../../utilities/templates")
+const env = require("../../environment")
 
+// development flag, can be used to test against templates exported locally
 const DEFAULT_TEMPLATES_BUCKET =
   "prod-budi-templates.s3-eu-west-1.amazonaws.com"
 
 exports.fetch = async function(ctx) {
   const { type = "app" } = ctx.query
 
-  const response = await fetch(
-    `https://${DEFAULT_TEMPLATES_BUCKET}/manifest.json`
-  )
-  const json = await response.json()
-  ctx.body = Object.values(json.templates[type])
+  if (env.LOCAL_TEMPLATES) {
+    ctx.body = Object.values(getLocalTemplates()[type])
+  } else {
+    const response = await fetch(
+      `https://${DEFAULT_TEMPLATES_BUCKET}/manifest.json`
+    )
+    const json = await response.json()
+    ctx.body = Object.values(json.templates[type])
+  }
 }
 
 exports.downloadTemplate = async function(ctx) {
   const { type, name } = ctx.params
 
-  await downloadTemplate(type, name)
+  if (!env.LOCAL_TEMPLATES) {
+    await downloadTemplate(type, name)
+  }
 
   ctx.body = {
     message: `template ${type}:${name} downloaded successfully.`,

packages/server/src/api/controllers/user.js

@@ -37,15 +37,22 @@ exports.create = async function(ctx) {
     accessLevelId,
   }
 
-  const response = await db.post(user)
-
-  ctx.status = 200
-  ctx.message = "User created successfully."
-  ctx.userId = response._id
-  ctx.body = {
-    _rev: response.rev,
-    username,
-    name,
+  try {
+    const response = await db.post(user)
+    ctx.status = 200
+    ctx.message = "User created successfully."
+    ctx.userId = response._id
+    ctx.body = {
+      _rev: response.rev,
+      username,
+      name,
+    }
+  } catch (err) {
+    if (err.status === 409) {
+      ctx.throw(400, "User exists already")
+    } else {
+      ctx.throw(err.status, err)
+    }
   }
 }
 

packages/server/src/api/routes/pages.old.js

@@ -1,111 +0,0 @@
-const Router = require("@koa/router")
-const StatusCodes = require("../../utilities/statusCodes")
-const joiValidator = require("../../middleware/joi-validator")
-const Joi = require("joi")
-const {
-  listScreens,
-  saveScreen,
-  compileStaticAssetsForPage,
-  deleteScreen,
-} = require("../../utilities/builder")
-const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/accessLevels")
-
-const router = Router()
-
-function generateSaveValidation() {
-  // prettier-ignore
-  return joiValidator.body(Joi.object({
-    _css: Joi.string().allow(""),
-    name: Joi.string().required(),
-    route: Joi.string().required(),
-    props: Joi.object({
-      _id: Joi.string().required(),
-      _component: Joi.string().required(),
-      _children: Joi.array().required(),
-      _instanceName: Joi.string().required(),
-      _styles: Joi.object().required(),
-      type: Joi.string().optional(),
-      table: Joi.string().optional(),
-    }).required().unknown(true),
-  }).unknown(true))
-}
-
-function generatePatchValidation() {
-  return joiValidator.body(
-    Joi.object({
-      oldname: Joi.string().required(),
-      newname: Joi.string().required(),
-    }).unknown(true)
-  )
-}
-
-router.post(
-  "/_builder/api/:appId/pages/:pageName",
-  authorized(BUILDER),
-  async ctx => {
-    await compileStaticAssetsForPage(
-      ctx.params.appId,
-      ctx.params.pageName,
-      ctx.request.body
-    )
-    ctx.response.status = StatusCodes.OK
-  }
-)
-
-router.get(
-  "/_builder/api/:appId/pages/:pagename/screens",
-  authorized(BUILDER),
-  async ctx => {
-    ctx.body = await listScreens(ctx.params.appId, ctx.params.pagename)
-    ctx.response.status = StatusCodes.OK
-  }
-)
-
-router.post(
-  "/_builder/api/:appId/pages/:pagename/screen",
-  authorized(BUILDER),
-  generateSaveValidation(),
-  async ctx => {
-    ctx.body = await saveScreen(
-      ctx.config,
-      ctx.params.appId,
-      ctx.params.pagename,
-      ctx.request.body
-    )
-    ctx.response.status = StatusCodes.OK
-  }
-)
-
-router.patch(
-  "/_builder/api/:appname/pages/:pagename/screen",
-  authorized(BUILDER),
-  generatePatchValidation(),
-  async ctx => {
-    await renameScreen(
-      ctx.config,
-      ctx.params.appname,
-      ctx.params.pagename,
-      ctx.request.body.oldname,
-      ctx.request.body.newname
-    )
-    ctx.response.status = StatusCodes.OK
-  }
-)
-
-router.delete(
-  "/_builder/api/pages/:pagename/screens/:id",
-  authorized(BUILDER),
-  async ctx => {
-    await deleteScreen(
-      ctx.config,
-      ctx.user.appId,
-      ctx.params.pagename,
-      ctx.params.id
-    )
-
-    ctx.response.status = StatusCodes.OK
-  }
-)
-
-module.exports = router

packages/server/src/environment.js

@@ -34,6 +34,7 @@ module.exports = {
   USERID_API_KEY: process.env.USERID_API_KEY,
   ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
   DEPLOYMENT_DB_URL: process.env.DEPLOYMENT_DB_URL,
+  LOCAL_TEMPLATES: process.env.LOCAL_TEMPLATES,
   _set(key, value) {
     process.env[key] = value
     module.exports[key] = value

packages/server/src/utilities/templates.js

@@ -8,12 +8,35 @@ const zlib = require("zlib")
 const { promisify } = require("util")
 const streamPipeline = promisify(stream.pipeline)
 const { budibaseAppsDir } = require("./budibaseDir")
+const env = require("../environment")
 const CouchDB = require("../db")
+const { DocumentTypes } = require("../db/utils")
 
 const DEFAULT_TEMPLATES_BUCKET =
   "prod-budi-templates.s3-eu-west-1.amazonaws.com"
 
+exports.getLocalTemplates = function() {
+  const templatesDir = join(os.homedir(), ".budibase", "templates", "app")
+  const templateObj = { app: {} }
+  fs.ensureDirSync(templatesDir)
+  const templateNames = fs.readdirSync(templatesDir)
+  for (let name of templateNames) {
+    templateObj.app[name] = {
+      name,
+      category: "local",
+      description: "local template",
+      type: "app",
+      key: `app/${name}`,
+    }
+  }
+  return templateObj
+}
+
 exports.downloadTemplate = async function(type, name) {
+  const dirName = join(budibaseAppsDir(), "templates", type, name)
+  if (env.LOCAL_TEMPLATES) {
+    return dirName
+  }
   const templateUrl = `https://${DEFAULT_TEMPLATES_BUCKET}/templates/${type}/${name}.tar.gz`
   const response = await fetch(templateUrl)
 
@@ -30,26 +53,27 @@ exports.downloadTemplate = async function(type, name) {
     tar.extract(join(budibaseAppsDir(), "templates", type))
   )
 
-  return join(budibaseAppsDir(), "templates", type, name)
+  return dirName
 }
 
 exports.exportTemplateFromApp = async function({ templateName, appId }) {
   // Copy frontend files
-  const appToExport = join(os.homedir(), ".budibase", appId, "pages")
-  const templatesDir = join(os.homedir(), ".budibase", "templates")
-  fs.ensureDirSync(templatesDir)
-
-  const templateOutputPath = join(templatesDir, templateName)
-  fs.copySync(appToExport, join(templateOutputPath, "pages"))
-
-  fs.ensureDirSync(join(templateOutputPath, "db"))
-  const writeStream = fs.createWriteStream(
-    join(templateOutputPath, "db", "dump.txt")
+  const templatesDir = join(
+    os.homedir(),
+    ".budibase",
+    "templates",
+    "app",
+    templateName,
+    "db"
   )
-
+  fs.ensureDirSync(templatesDir)
+  const writeStream = fs.createWriteStream(join(templatesDir, "dump.txt"))
   // perform couch dump
   const instanceDb = new CouchDB(appId)
-
-  await instanceDb.dump(writeStream)
-  return templateOutputPath
+  await instanceDb.dump(writeStream, {
+    filter: doc => {
+      return !doc._id.startsWith(DocumentTypes.USER)
+    },
+  })
+  return templatesDir
 }