Merge branch 'develop' of github.com:Budibase/budibase into lab-day/refactor-app-db

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.0.49-alpha.2",
+  "version": "1.0.49-alpha.5",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -36,10 +36,10 @@
     "dev:server": "lerna run --parallel dev:builder --concurrency 1 --scope @budibase/worker --scope @budibase/server",
     "test": "lerna run test",
     "lint:eslint": "eslint packages",
-    "lint:prettier": "prettier --check \"packages/**/*.{js,svelte}\"",
+    "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\"",
     "lint": "yarn run lint:eslint && yarn run lint:prettier",
     "lint:fix:eslint": "eslint --fix packages",
-    "lint:fix:prettier": "prettier --write \"packages/**/*.{js,svelte}\"",
+    "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\"",
     "lint:fix:ts": "lerna run lint:fix",
     "lint:fix": "yarn run lint:fix:ts && yarn run lint:fix:prettier && yarn run lint:fix:eslint",
     "test:e2e": "lerna run cy:test",

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "1.0.49-alpha.2",
+  "version": "1.0.49-alpha.5",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "src/index.js",
   "author": "Budibase",

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "1.0.49-alpha.2",
+  "version": "1.0.49-alpha.5",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",

packages/builder/cypress/ts/setup.ts

@@ -1,4 +1,4 @@
 // @ts-ignore
 import { run } from "../setup"
 
-run("../../server/src/index")
+run("../../server/src/index")

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "1.0.49-alpha.2",
+  "version": "1.0.49-alpha.5",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -66,10 +66,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.49-alpha.2",
-    "@budibase/client": "^1.0.49-alpha.2",
+    "@budibase/bbui": "^1.0.49-alpha.5",
+    "@budibase/client": "^1.0.49-alpha.5",
     "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^1.0.49-alpha.2",
+    "@budibase/string-templates": "^1.0.49-alpha.5",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "1.0.49-alpha.2",
+  "version": "1.0.49-alpha.5",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "1.0.49-alpha.2",
+  "version": "1.0.49-alpha.5",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.49-alpha.2",
+    "@budibase/bbui": "^1.0.49-alpha.5",
     "@budibase/standard-components": "^0.9.139",
-    "@budibase/string-templates": "^1.0.49-alpha.2",
+    "@budibase/string-templates": "^1.0.49-alpha.5",
     "regexparam": "^1.3.0",
     "rollup-plugin-polyfill-node": "^0.8.0",
     "shortid": "^2.2.15",

packages/client/src/utils/fetch/QueryFetch.js

@@ -16,7 +16,12 @@ export default class QueryFetch extends DataFetch {
     if (!datasource?._id) {
       return null
     }
-    return await fetchQueryDefinition(datasource._id)
+    const definition = await fetchQueryDefinition(datasource._id)
+    // After getting the definition of query, it loses "fields" attribute because of security reason from the server. However, this attribute needs to be inside of defintion for pagination.
+    if (!definition.fields) {
+      definition.fields = datasource.fields
+    }
+    return definition
   }
 
   async getData() {

packages/server/__mocks__/arangojs.ts

@@ -9,7 +9,7 @@ module ArangoMock {
     this.close = jest.fn()
   }
 
-// @ts-ignore
+  // @ts-ignore
   arangodb.aql = (strings, ...args) => {
     let str = strings.join("{}")
 

packages/server/__mocks__/aws-sdk.ts

@@ -1,7 +1,7 @@
 module AwsMock {
   const aws: any = {}
 
-  const response = (body: any) => () => ({promise: () => body})
+  const response = (body: any) => () => ({ promise: () => body })
 
   function DocumentClient() {
     // @ts-ignore
@@ -39,9 +39,9 @@ module AwsMock {
     )
   }
 
-  aws.DynamoDB = {DocumentClient}
+  aws.DynamoDB = { DocumentClient }
   aws.S3 = S3
-  aws.config = {update: jest.fn()}
+  aws.config = { update: jest.fn() }
 
   module.exports = aws
-}
+}

packages/server/__mocks__/mongodb.ts

@@ -5,15 +5,14 @@ module MongoMock {
     this.connect = jest.fn()
     this.close = jest.fn()
     this.insertOne = jest.fn()
-    this.insertMany = jest.fn(() => ({toArray: () => []}))
-    this.find = jest.fn(() => ({toArray: () => []}))
+    this.insertMany = jest.fn(() => ({ toArray: () => [] }))
+    this.find = jest.fn(() => ({ toArray: () => [] }))
     this.findOne = jest.fn()
     this.count = jest.fn()
     this.deleteOne = jest.fn()
-    this.deleteMany = jest.fn(() => ({toArray: () => []}))
+    this.deleteMany = jest.fn(() => ({ toArray: () => [] }))
     this.updateOne = jest.fn()
-    this.updateMany = jest.fn(() => ({toArray: () => []}))
-
+    this.updateMany = jest.fn(() => ({ toArray: () => [] }))
 
     this.collection = jest.fn(() => ({
       insertOne: this.insertOne,

packages/server/__mocks__/mssql.ts

@@ -10,15 +10,15 @@ module MsSqlMock {
     ],
   }))
 
-// mssql.connect = jest.fn(() => ({ recordset: [] }))
+  // mssql.connect = jest.fn(() => ({ recordset: [] }))
 
   mssql.ConnectionPool = jest.fn(() => ({
     connect: jest.fn(() => ({
       request: jest.fn(() => ({
-        query: jest.fn(sql => ({ recordset: [ sql ] })),
+        query: jest.fn(sql => ({ recordset: [sql] })),
       })),
     })),
   }))
 
   module.exports = mssql
-}
+}

packages/server/__mocks__/node-fetch.ts

@@ -62,7 +62,8 @@ module FetchMock {
       return json({
         url,
         opts,
-        value: "<!doctype html><html itemscope=\"\" itemtype=\"http://schema.org/WebPage\" lang=\"en-GB\"></html>",
+        value:
+          '<!doctype html><html itemscope="" itemtype="http://schema.org/WebPage" lang="en-GB"></html>',
       })
     } else if (url.includes("failonce.com")) {
       failCount++

packages/server/__mocks__/pg.ts

@@ -10,16 +10,14 @@ module PgMock {
     ],
   }))
 
-// constructor
-  function Client() {
-  }
+  // constructor
+  function Client() {}
 
   Client.prototype.query = query
   Client.prototype.connect = jest.fn()
   Client.prototype.release = jest.fn()
 
-  function Pool() {
-  }
+  function Pool() {}
 
   const on = jest.fn()
   Pool.prototype.query = query

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.49-alpha.2",
+  "version": "1.0.49-alpha.5",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -70,9 +70,9 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.49-alpha.2",
-    "@budibase/client": "^1.0.49-alpha.2",
-    "@budibase/string-templates": "^1.0.49-alpha.2",
+    "@budibase/backend-core": "^1.0.49-alpha.5",
+    "@budibase/client": "^1.0.49-alpha.5",
+    "@budibase/string-templates": "^1.0.49-alpha.5",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",

packages/server/scripts/likeCypress.ts

@@ -30,4 +30,4 @@ process.env.ALLOW_DEV_AUTOMATIONS = "1"
 const server = require("../src/app")
 process.env.PORT = WORKER_PORT
 const worker = require("../../worker/src/index")
-process.env.PORT = MAIN_PORT
+process.env.PORT = MAIN_PORT

packages/server/src/migrations/functions/appUrls.ts

@@ -9,7 +9,16 @@ import { getAppUrl } from "../../api/controllers/application"
  * Add the url to the app metadata if it doesn't exist
  */
 export const run = async (appDb: any) => {
-  const metadata = await appDb.get(DocumentTypes.APP_METADATA)
+  let metadata
+  try {
+    metadata = await appDb.get(DocumentTypes.APP_METADATA)
+  } catch (e) {
+    // sometimes the metadata document doesn't exist
+    // exit early instead of failing the migration
+    console.error("Error retrieving app metadata. Skipping", e)
+    return
+  }
+
   if (!metadata.url) {
     const context = {
       request: {
@@ -20,6 +29,6 @@ export const run = async (appDb: any) => {
     }
     metadata.url = getAppUrl(context)
     console.log(`Adding url to app: ${metadata.url}`)
+    await appDb.put(metadata)
   }
-  await appDb.put(metadata)
 }

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "1.0.49-alpha.2",
+  "version": "1.0.49-alpha.5",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/worker/.eslintrc

@@ -3,10 +3,11 @@
     "emit": true,
     "key": true
   },
+  "parser": "@typescript-eslint/parser",
   "env": {
     "node": true
   },
   "extends": ["eslint:recommended"],
   "rules": {
   }
-}
+}

packages/worker/.gitignore

@@ -1,3 +1,4 @@
 node_modules/
 .env
-watchtower-hook.json
+watchtower-hook.json
+dist/

packages/worker/nodemon.json

@@ -1,3 +1,6 @@
 {
-  "watch": ["src", "../backend-core"]
-}
+  "watch": ["src", "../backend-core"],
+  "ext": "js,ts,json",
+  "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js"],
+  "exec": "ts-node src/index.ts"
+}

packages/worker/package.json

@@ -1,9 +1,9 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "1.0.49-alpha.2",
+  "version": "1.0.49-alpha.5",
   "description": "Budibase background service",
-  "main": "src/index.js",
+  "main": "src/index.ts",
   "repository": {
     "type": "git",
     "url": "https://github.com/Budibase/budibase.git"
@@ -12,10 +12,15 @@
     "budibase"
   ],
   "scripts": {
-    "run:docker": "node src/index.js",
+    "build": "rimraf dist/ && tsc",
+    "postbuild": "copyfiles -u 1 src/**/*.hbs dist/",
+    "run:docker": "node dist/index.js",
     "build:docker": "docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION",
     "dev:stack:init": "node ./scripts/dev/manage.js init",
     "dev:builder": "npm run dev:stack:init && nodemon",
+    "format": "prettier --config ../../.prettierrc.json 'src/**/*.ts' --write",
+    "lint": "eslint --fix src/",
+    "lint:fix": "yarn run format && yarn run lint",
     "test": "jest --runInBand",
     "env:multi:enable": "node scripts/multiTenancy.js enable",
     "env:multi:disable": "node scripts/multiTenancy.js disable",
@@ -29,8 +34,8 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "^1.0.49-alpha.2",
-    "@budibase/string-templates": "^1.0.49-alpha.2",
+    "@budibase/backend-core": "^1.0.49-alpha.5",
+    "@budibase/string-templates": "^1.0.49-alpha.5",
     "@koa/router": "^8.0.0",
     "@sentry/node": "^6.0.0",
     "@techpass/passport-openidconnect": "^0.3.0",
@@ -58,13 +63,26 @@
     "server-destroy": "^1.0.1"
   },
   "devDependencies": {
-    "jest": "^26.6.3",
+    "@types/jest": "^26.0.23",
+    "@types/koa": "^2.13.3",
+    "@types/koa-router": "^7.4.2",
+    "@types/node": "^15.12.4",
+    "@typescript-eslint/parser": "4.28.0",
+    "copyfiles": "^2.4.1",
+    "eslint": "^6.8.0",
+    "jest": "^27.0.5",
     "nodemon": "^2.0.7",
     "pouchdb-adapter-memory": "^7.2.2",
+    "prettier": "2.3.1",
+    "rimraf": "^3.0.2",
     "supertest": "^6.1.3",
+    "ts-jest": "^27.0.3",
+    "ts-node": "^10.0.0",
+    "typescript": "4.3.5",
     "update-dotenv": "^1.1.1"
   },
   "jest": {
+    "preset": "ts-jest",
     "testEnvironment": "node",
     "setupFiles": [
       "./scripts/jestSetup.js"

packages/worker/src/api/routes/tests/auth.spec.js

@@ -1,10 +1,9 @@
+jest.mock("nodemailer")
 const setup = require("./utilities")
+const sendMailMock = setup.emailMock()
 
 const TENANT_ID = "default"
 
-jest.mock("nodemailer")
-const sendMailMock = setup.emailMock()
-
 describe("/api/global/auth", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()

packages/worker/src/api/routes/tests/configs.spec.js

@@ -1,11 +1,7 @@
-const setup = require("./utilities")
-
 // mock the email system
 jest.mock("nodemailer")
-const nodemailer = require("nodemailer")
-nodemailer.createTransport.mockReturnValue({
-  verify: jest.fn(),
-})
+const setup = require("./utilities")
+setup.emailMock()
 
 describe("/api/global/configs/checklist", () => {
   let request = setup.getRequest()

packages/worker/src/api/routes/tests/email.spec.js

@@ -1,16 +1,10 @@
+jest.mock("nodemailer")
 const setup = require("./utilities")
+const sendMailMock = setup.emailMock()
+
 const { EmailTemplatePurpose } = require("../../../constants")
 const { TENANT_ID } = require("./utilities/structures")
 
-// mock the email system
-const sendMailMock = jest.fn()
-jest.mock("nodemailer")
-const nodemailer = require("nodemailer")
-nodemailer.createTransport.mockReturnValue({
-  sendMail: sendMailMock,
-  verify: jest.fn(),
-})
-
 describe("/api/global/email", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()

packages/worker/src/api/routes/tests/users.spec.js

@@ -1,7 +1,5 @@
-const setup = require("./utilities")
-const { TENANT_ID } = require("./utilities/structures")
-
 jest.mock("nodemailer")
+const setup = require("./utilities")
 const sendMailMock = setup.emailMock()
 
 describe("/api/global/users", () => {

packages/worker/src/index.js → packages/worker/src/index.ts

@@ -1,4 +1,8 @@
 // need to load environment first
+import { Scope } from "@sentry/node"
+import { Event } from "@sentry/types/dist/event"
+import Application from "koa"
+
 const env = require("./environment")
 const CouchDB = require("./db")
 require("@budibase/backend-core").init(CouchDB)
@@ -13,7 +17,7 @@ const api = require("./api")
 const redis = require("./utilities/redis")
 const Sentry = require("@sentry/node")
 
-const app = new Koa()
+const app: Application = new Koa()
 
 app.keys = ["secret", "key"]
 
@@ -42,8 +46,8 @@ if (env.isProd()) {
   Sentry.init()
 
   app.on("error", (err, ctx) => {
-    Sentry.withScope(function (scope) {
-      scope.addEventProcessor(function (event) {
+    Sentry.withScope(function (scope: Scope) {
+      scope.addEventProcessor(function (event: Event) {
         return Sentry.Handlers.parseRequest(event, ctx.request)
       })
       Sentry.captureException(err)

packages/worker/tsconfig.json

@@ -0,0 +1,23 @@
+{
+  "compilerOptions": {
+    "target": "es6",
+    "module": "commonjs",
+    "lib": ["es2019"],
+    "allowJs": true,
+    "outDir": "dist",
+    "strict": true,
+    "noImplicitAny": true,
+    "esModuleInterop": true,
+    "resolveJsonModule": true,
+    "incremental": true
+  },
+  "include": [
+    "./src/**/*"
+  ],
+  "exclude": [
+    "node_modules",
+    "**/*.json",
+    "**/*.spec.ts",
+    "**/*.spec.js"
+  ]
+}