pages being stored in couch on initialisation and page save

packages/builder/src/builderStore/storeUtils.js

@@ -38,7 +38,7 @@ export const saveCurrentPreviewItem = s =>
 export const savePage = async s => {
   const pageName = s.currentPageName || "main"
   const page = s.pages[pageName]
-  await api.post(`/_builder/api/${s.appId}/pages/${pageName}`, {
+  await api.post(`/api/pages/${pageName}`, {
     page: { componentLibraries: s.pages.componentLibraries, ...page },
     uiFunctions: s.currentPageFunctions,
     screens: page._screens,

packages/server/src/api/controllers/application.js

@@ -1,5 +1,5 @@
 const CouchDB = require("../../db")
-const { getPackageForBuilder, buildPage } = require("../../utilities/builder")
+const { buildPage } = require("../../utilities/builder")
 const env = require("../../environment")
 const { copy, existsSync, readFile, writeFile } = require("fs-extra")
 const { budibaseAppsDir } = require("../../utilities/budibaseDir")
@@ -12,10 +12,17 @@ const chmodr = require("chmodr")
 const packageJson = require("../../../package.json")
 const { createLinkView } = require("../../db/linkedRows")
 const { downloadTemplate } = require("../../utilities/templates")
-const { generateAppID, DocumentTypes, SEPARATOR } = require("../../db/utils")
+const {
+  generateAppID,
+  DocumentTypes,
+  SEPARATOR,
+  getPageParams,
+  generatePageID,
+} = require("../../db/utils")
 const {
   downloadExtractComponentLibraries,
 } = require("../../utilities/createAppPackage")
+
 const APP_PREFIX = DocumentTypes.APP + SEPARATOR
 
 async function createInstance(template) {
@@ -60,7 +67,19 @@ exports.fetch = async function(ctx) {
 exports.fetchAppPackage = async function(ctx) {
   const db = new CouchDB(ctx.params.appId)
   const application = await db.get(ctx.params.appId)
-  ctx.body = await getPackageForBuilder(ctx.config, application)
+  // ctx.body = await getPackageForBuilder(application)
+
+  const pages = await db.allDocs(
+    getPageParams(null, {
+      include_docs: true,
+    })
+  )
+
+  ctx.body = {
+    application,
+    pages,
+  }
+
   setBuilderToken(ctx, ctx.params.appId, application.version)
 }
 
@@ -132,6 +151,8 @@ const createEmptyAppPackage = async (ctx, app) => {
   const appsFolder = budibaseAppsDir()
   const newAppFolder = resolve(appsFolder, app._id)
 
+  const db = new CouchDB(app._id)
+
   if (existsSync(newAppFolder)) {
     ctx.throw(400, "App folder already exists for this application")
   }
@@ -164,6 +185,8 @@ const createEmptyAppPackage = async (ctx, app) => {
       app.template.key,
       "pages"
     )
+    // TODO: write the template page JSON to couch
+    // iterate over the pages and write them to the db
     await copy(templatePageDefinitions, join(appsFolder, app._id, "pages"))
   }
 
@@ -172,7 +195,10 @@ const createEmptyAppPackage = async (ctx, app) => {
     app
   )
 
-  await buildPage(ctx.config, app._id, "main", {
+  mainJson._id = generatePageID()
+  await db.put(mainJson)
+
+  await buildPage(app._id, "main", {
     page: mainJson,
     screens: await loadScreens(newAppFolder, "main"),
   })
@@ -182,7 +208,11 @@ const createEmptyAppPackage = async (ctx, app) => {
     app
   )
 
-  await buildPage(ctx.config, app._id, "unauthenticated", {
+  // Write to couch
+  unauthenticatedJson._id = generatePageID()
+  await db.put(unauthenticatedJson)
+
+  await buildPage(app._id, "unauthenticated", {
     page: unauthenticatedJson,
     screens: await loadScreens(newAppFolder, "unauthenticated"),
   })

packages/server/src/api/controllers/page.js

@@ -0,0 +1,33 @@
+const CouchDB = require("../../db/client")
+const { generatePageID } = require("../../db/utils")
+const { buildPage } = require("../../utilities/builder")
+
+exports.save = async function(ctx) {
+  const db = new CouchDB(ctx.user.appId)
+
+  const appPackage = ctx.request.body
+
+  // TODO: rename to something more descriptive
+  await buildPage(
+    ctx.config,
+    ctx.user.appId,
+    ctx.params.pageName,
+    ctx.request.body
+  )
+
+  // write the page data to couchDB
+  // if (pkg.page._css) {
+  //   delete pkg.page._css
+  // }
+
+  // if (pkg.page.name) {
+  //   delete pkg.page.name
+  // }
+
+  // if (pkg.page._screens) {
+  //   delete pkg.page._screens
+  // }
+  appPackage.page._id = appPackage.page._id || generatePageID()
+  await db.put(appPackage.page)
+  ctx.response.status = 200
+}

packages/server/src/api/routes/pages.js

@@ -45,12 +45,7 @@ router.post(
   "/_builder/api/:appId/pages/:pageName",
   authorized(BUILDER),
   async ctx => {
-    await buildPage(
+    await buildPage(ctx.params.appId, ctx.params.pageName, ctx.request.body)
-      ctx.config,
-      ctx.params.appId,
-      ctx.params.pageName,
-      ctx.request.body
-    )
     ctx.response.status = StatusCodes.OK
   }
 )
@@ -59,11 +54,7 @@ router.get(
   "/_builder/api/:appId/pages/:pagename/screens",
   authorized(BUILDER),
   async ctx => {
-    ctx.body = await listScreens(
+    ctx.body = await listScreens(ctx.params.appId, ctx.params.pagename)
-      ctx.config,
-      ctx.params.appId,
-      ctx.params.pagename
-    )
     ctx.response.status = StatusCodes.OK
   }
 )

packages/server/src/api/routes/pages.new.js

@@ -0,0 +1,35 @@
+const Router = require("@koa/router")
+const joiValidator = require("../../middleware/joi-validator")
+const Joi = require("joi")
+const authorized = require("../../middleware/authorized")
+const { BUILDER } = require("../../utilities/accessLevels")
+const controller = require("../controllers/page")
+
+const router = Router()
+
+function generateSaveValidation() {
+  // prettier-ignore
+  return joiValidator.body(Joi.object({
+    _css: Joi.string().allow(""),
+    name: Joi.string().required(),
+    route: Joi.string().required(),
+    props: Joi.object({
+      _id: Joi.string().required(),
+      _component: Joi.string().required(),
+      _children: Joi.array().required(),
+      _instanceName: Joi.string().required(),
+      _styles: Joi.object().required(),
+      type: Joi.string().optional(),
+      table: Joi.string().optional(),
+    }).required().unknown(true),
+  }).unknown(true))
+}
+
+router.post(
+  "/api/pages/:pageName",
+  authorized(BUILDER),
+  generateSaveValidation(),
+  controller.save
+)
+
+module.exports = router

packages/server/src/db/client.js

@@ -26,4 +26,18 @@ const Pouch = PouchDB.defaults(POUCH_DB_DEFAULTS)
 
 allDbs(Pouch)
 
+// replicate your local levelDB pouch to a running HTTP compliant couch or pouchdb server.
+// eslint-disable-next-line no-unused-vars
+function replicateLocal() {
+  Pouch.allDbs().then(dbs => {
+    for (let db of dbs) {
+      new Pouch(db).sync(
+        new PouchDB(`http://127.0.0.1:5984/${db}`, { live: true })
+      )
+    }
+  })
+}
+
+replicateLocal()
+
 module.exports = Pouch

packages/server/src/db/utils.js

@@ -13,6 +13,8 @@ const DocumentTypes = {
   ACCESS_LEVEL: "ac",
   WEBHOOK: "wh",
   INSTANCE: "inst",
+  PAGE: "page",
+  SCREEN: "screen",
 }
 
 exports.DocumentTypes = DocumentTypes
@@ -175,6 +177,29 @@ exports.generateWebhookID = () => {
   return `${DocumentTypes.WEBHOOK}${SEPARATOR}${newid()}`
 }
 
+/**
+ * Generates a new screen ID.
+ * @returns {string} The new screen ID which the screen doc can be stored under.
+ */
+exports.generateScreenID = () => {
+  return `${DocumentTypes.SCREEN}${SEPARATOR}${newid()}`
+}
+
+/**
+ * Generates a new page ID.
+ * @returns {string} The new page ID which the page doc can be stored under.
+ */
+exports.generatePageID = () => {
+  return `${DocumentTypes.PAGE}${SEPARATOR}${newid()}`
+}
+
+/**
+ * Gets parameters for retrieving pages, this is a utility function for the getDocParams function.
+ */
+exports.getPageParams = (pageId = null, otherProps = {}) => {
+  return getDocParams(DocumentTypes.PAGE, pageId, otherProps)
+}
+
 /**
  * Gets parameters for retrieving a webhook, this is a utility function for the getDocParams function.
  */

packages/server/src/utilities/builder/buildPage.js

@@ -1,33 +1,27 @@
-const { appPackageFolder } = require("../createAppPackage")
+const { constants, copyFile, writeFile, readFile } = require("fs-extra")
-const {
-  constants,
-  copyFile,
-  writeFile,
-  readFile,
-  writeJSON,
-} = require("fs-extra")
 const { join, resolve } = require("../centralPath")
 const sqrl = require("squirrelly")
 const { convertCssToFiles } = require("./convertCssToFiles")
 const publicPath = require("./publicPath")
+const { budibaseAppsDir } = require("../budibaseDir")
 
-module.exports = async (config, appId, pageName, pkg) => {
+module.exports = async (appId, pageName, pkg) => {
-  const appPath = appPackageFolder(config, appId)
+  const appPath = join(budibaseAppsDir(), appId)
 
   pkg.screens = pkg.screens || []
 
   await convertCssToFiles(publicPath(appPath, pageName), pkg)
 
-  await buildIndexHtml(config, appId, pageName, appPath, pkg)
+  await buildIndexHtml(appId, pageName, appPath, pkg)
 
-  await buildFrontendAppDefinition(config, appId, pageName, pkg, appPath)
+  await buildFrontendAppDefinition(appId, pageName, pkg, appPath)
 
   await copyClientLib(appPath, pageName)
 
-  await savePageJson(appPath, pageName, pkg)
+  // await savePageJson(appPath, pageName, pkg)
 }
 
-const rootPath = (config, appId) => (config.useAppRootPath ? `/${appId}` : "")
+// const rootPath = (config, appId) => (config.useAppRootPath ? `/${appId}` : "")
 
 const copyClientLib = async (appPath, pageName) => {
   const sourcepath = require.resolve("@budibase/client")
@@ -42,11 +36,10 @@ const copyClientLib = async (appPath, pageName) => {
   )
 }
 
-const buildIndexHtml = async (config, appId, pageName, appPath, pkg) => {
+const buildIndexHtml = async (appId, pageName, appPath, pkg) => {
   const appPublicPath = publicPath(appPath, pageName)
 
-  const stylesheetUrl = s =>
+  const stylesheetUrl = s => (s.startsWith("http") ? s : `/${appId}/${s}`)
-    s.startsWith("http") ? s : `/${rootPath(config, appId)}/${s}`
 
   const templateObj = {
     title: pkg.page.title || "Budibase App",
@@ -76,12 +69,13 @@ const buildIndexHtml = async (config, appId, pageName, appPath, pkg) => {
   await writeFile(deployableHtmlPath, deployableHtml, { flag: "w+" })
 }
 
-const buildFrontendAppDefinition = async (config, appId, pageName, pkg) => {
+const buildFrontendAppDefinition = async (appId, pageName, pkg) => {
-  const appPath = appPackageFolder(config, appId)
+  const appPath = join(budibaseAppsDir(), appId)
   const appPublicPath = publicPath(appPath, pageName)
 
   const filename = join(appPublicPath, "clientFrontendDefinition.js")
 
+  // TODO: weird - why
   if (pkg.page._css) {
     delete pkg.page._css
   }
@@ -106,22 +100,22 @@ const buildFrontendAppDefinition = async (config, appId, pageName, pkg) => {
   )
 }
 
-const savePageJson = async (appPath, pageName, pkg) => {
+// const savePageJson = async (appPath, pageName, pkg) => {
-  const pageFile = join(appPath, "pages", pageName, "page.json")
+//   const pageFile = join(appPath, "pages", pageName, "page.json")
 
-  if (pkg.page._css) {
+//   if (pkg.page._css) {
-    delete pkg.page._css
+//     delete pkg.page._css
-  }
+//   }
 
-  if (pkg.page.name) {
+//   if (pkg.page.name) {
-    delete pkg.page.name
+//     delete pkg.page.name
-  }
+//   }
 
-  if (pkg.page._screens) {
+//   if (pkg.page._screens) {
-    delete pkg.page._screens
+//     delete pkg.page._screens
-  }
+//   }
 
-  await writeJSON(pageFile, pkg.page, {
+//   await writeJSON(pageFile, pkg.page, {
-    spaces: 2,
+//     spaces: 2,
-  })
+//   })
-}
+// }

packages/server/src/utilities/builder/index.js

@@ -8,36 +8,37 @@ const {
   unlink,
   rmdir,
 } = require("fs-extra")
-const { join, resolve } = require("../centralPath")
+const { join } = require("../centralPath")
 const { dirname } = require("path")
 
 const buildPage = require("./buildPage")
-const getPages = require("./getPages")
+// const getPages = require("./getPages")
 const listScreens = require("./listScreens")
-const deleteCodeMeta = require("./deleteCodeMeta")
+const { budibaseAppsDir } = require("../budibaseDir")
+// const { budibaseAppsDir } = require("../budibaseDir")
 
 module.exports.buildPage = buildPage
 module.exports.listScreens = listScreens
 
-const getAppDefinition = async appPath =>
+// const getAppDefinition = async appPath =>
-  await readJSON(`${appPath}/appDefinition.json`)
+//   await readJSON(`${appPath}/appDefinition.json`)
 
-module.exports.getPackageForBuilder = async (config, application) => {
+// module.exports.getPackageForBuilder = async application => {
-  const appPath = resolve(config.latestPackagesFolder, application._id)
+//   const appPath = resolve(budibaseAppsDir(), application._id)
 
-  const pages = await getPages(appPath)
+//   const pages = await getPages(appPath)
 
-  return {
+//   return {
-    pages,
+//     pages,
-    application,
+//     application,
-  }
+//   }
-}
+// }
 
 const screenPath = (appPath, pageName, name) =>
   join(appPath, "pages", pageName, "screens", name + ".json")
 
-module.exports.saveScreen = async (config, appname, pagename, screen) => {
+module.exports.saveScreen = async (appId, pagename, screen) => {
-  const appPath = appPackageFolder(config, appname)
+  const appPath = join(budibaseAppsDir(), appId)
   const compPath = screenPath(appPath, pagename, screen.props._id)
 
   await ensureDir(dirname(compPath))
@@ -45,8 +46,6 @@ module.exports.saveScreen = async (config, appname, pagename, screen) => {
     delete screen._css
   }
 
-  deleteCodeMeta(screen.props)
-
   await writeJSON(compPath, screen, {
     encoding: "utf8",
     flag: "w",
@@ -57,12 +56,12 @@ module.exports.saveScreen = async (config, appname, pagename, screen) => {
 
 module.exports.renameScreen = async (
   config,
-  appname,
+  appId,
   pagename,
   oldName,
   newName
 ) => {
-  const appPath = appPackageFolder(config, appname)
+  const appPath = join(budibaseAppsDir(), appId)
 
   const oldComponentPath = screenPath(appPath, pagename, oldName)
 
@@ -73,7 +72,7 @@ module.exports.renameScreen = async (
 }
 
 module.exports.deleteScreen = async (config, appId, pagename, name) => {
-  const appPath = appPackageFolder(config, appId)
+  const appPath = join(budibaseAppsDir(), appId)
   const componentFile = screenPath(appPath, pagename, name)
   await unlink(componentFile)
 
@@ -83,16 +82,16 @@ module.exports.deleteScreen = async (config, appId, pagename, name) => {
   }
 }
 
-module.exports.savePage = async (config, appname, pagename, page) => {
+// module.exports.savePage = async (appId, pagename, page) => {
-  const appPath = appPackageFolder(config, appname)
+//   const appPath = join(budibaseAppsDir(), appId)
-  const pageDir = join(appPath, "pages", pagename)
+//   const pageDir = join(appPath, "pages", pagename)
-
+
-  await ensureDir(pageDir)
+//   await ensureDir(pageDir)
-  await writeJSON(join(pageDir, "page.json"), page, {
+//   await writeJSON(join(pageDir, "page.json"), page, {
-    encoding: "utf8",
+//     encoding: "utf8",
-    flag: "w",
+//     flag: "w",
-    space: 2,
+//     space: 2,
-  })
+//   })
-  const appDefinition = await getAppDefinition(appPath)
+//   const appDefinition = await getAppDefinition(appPath)
-  await buildPage(config, appname, appDefinition, pagename, page)
+//   await buildPage(appId, appDefinition, pagename, page)
-}
+// }

packages/server/src/utilities/builder/listScreens.js

@@ -1,10 +1,10 @@
-const { appPackageFolder } = require("../createAppPackage")
 const { readJSON, readdir, stat } = require("fs-extra")
 const { join } = require("../centralPath")
 const { keyBy } = require("lodash/fp")
+const { budibaseAppsDir } = require("../budibaseDir")
 
-module.exports = async (config, appname, pagename) => {
+module.exports = async (appId, pagename) => {
-  const appPath = appPackageFolder(config, appname)
+  const appPath = join(budibaseAppsDir(), appId)
   return keyBy("name")(await fetchscreens(appPath, pagename))
 }
 

packages/server/src/utilities/createAppPackage.js

@@ -9,9 +9,6 @@ const packageJson = require("../../package.json")
 
 const streamPipeline = promisify(stream.pipeline)
 
-exports.appPackageFolder = (config, appname) =>
-  resolve(cwd(), config.latestPackagesFolder, appname)
-
 exports.downloadExtractComponentLibraries = async appFolder => {
   const LIBRARIES = ["standard-components"]