Merge branch 'master' of github.com:Budibase/budibase into develop

4 years ago · 3a340379e9
20 changed files with 156 additions and 50 deletions
--- a/hosting/docker-compose.yaml
+++ b/hosting/docker-compose.yaml
@ -75,7 +75,9 @@ services:
    ports:
      - "${MAIN_PORT}:10000"
    container_name: bbproxy
-    image: budibase/proxy
+    image: proxy-service
+    environment:
+      - PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10
    depends_on:
      - minio-service
      - worker-service
--- a/hosting/nginx.prod.conf.hbs
+++ b/hosting/nginx.prod.conf.hbs
@ -9,7 +9,11 @@ events {
 }

 http {
+  # rate limiting
+  limit_req_status 429;
  limit_req_zone $binary_remote_addr zone=ratelimit:10m rate=20r/s;
+  limit_req_zone $binary_remote_addr zone=webhooks:10m rate=${PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND}r/s;
+
  include       /etc/nginx/mime.types;
  default_type  application/octet-stream;
  proxy_set_header Host $host;
@ -126,6 +130,25 @@ http {
      proxy_pass      http://$apps:4002;
    }

+    location /api/webhooks/ {
+      # calls to webhooks are rate limited
+      limit_req zone=webhooks nodelay;
+
+      # Rest of configuration copied from /api/ location above
+      # 120s timeout on API requests
+      proxy_read_timeout 120s;
+      proxy_connect_timeout 120s;
+      proxy_send_timeout 120s;
+
+      proxy_http_version  1.1;
+      proxy_set_header    Connection          $connection_upgrade;
+      proxy_set_header    Upgrade             $http_upgrade;
+      proxy_set_header    X-Real-IP           $remote_addr;
+      proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+
+      proxy_pass      http://$apps:4002;
+    }
+
    location /db/ {
      proxy_pass      http://$couchdb:5984;
      rewrite ^/db/(.*)$ /$1 break;
--- a/hosting/proxy/Dockerfile
+++ b/hosting/proxy/Dockerfile
@ -1,3 +1,13 @@
 FROM nginx:latest
-COPY .generated-nginx.prod.conf /etc/nginx/nginx.conf
-COPY error.html /usr/share/nginx/html/error.html
+
+# nginx.conf
+# use the default nginx behaviour for *.template files which are processed with envsubst
+# override the output dir to output directly to /etc/nginx instead of /etc/nginx/conf.d
+ENV NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx
+COPY .generated-nginx.prod.conf /etc/nginx/templates/nginx.conf.template
+
+# Error handling
+COPY error.html /usr/share/nginx/html/error.html
+
+# Default environment
+ENV PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10
--- a/lerna.json
+++ b/lerna.json
@ -1,5 +1,5 @@
 {
-  "version": "1.2.26-alpha.0",
+  "version": "1.2.27",
  "npmClient": "yarn",
  "packages": [
    "packages/*"
--- a/packages/backend-core/package.json
+++ b/packages/backend-core/package.json
@ -1,6 +1,6 @@
 {
  "name": "@budibase/backend-core",
-  "version": "1.2.26-alpha.0",
+  "version": "1.2.27",
  "description": "Budibase backend core libraries used in server and worker",
  "main": "dist/src/index.js",
  "types": "dist/src/index.d.ts",
@ -20,7 +20,7 @@
    "test:watch": "jest --watchAll"
  },
  "dependencies": {
-    "@budibase/types": "1.2.26-alpha.0",
+    "@budibase/types": "^1.2.27",
    "@techpass/passport-openidconnect": "0.3.2",
    "aws-sdk": "2.1030.0",
    "bcrypt": "5.0.1",
--- a/packages/backend-core/src/events/emit/BBEventEmitter.ts
+++ b/packages/backend-core/src/events/emit/BBEventEmitter.ts
@ -0,0 +1,27 @@
+import { EventEmitter } from "events"
+import * as context from "../../context"
+import { Identity, Event } from "@budibase/types"
+
+export interface EmittedEvent {
+  tenantId: string
+  identity: Identity
+  appId: string | undefined
+  properties: any
+}
+
+class BBEventEmitter extends EventEmitter {
+  emitEvent(event: Event, properties: any, identity: Identity) {
+    const tenantId = context.getTenantId()
+    const appId = context.getAppId()
+
+    const emittedEvent: EmittedEvent = {
+      tenantId,
+      identity,
+      appId,
+      properties,
+    }
+    this.emit(event, emittedEvent)
+  }
+}
+
+export const emitter = new BBEventEmitter()
--- a/packages/backend-core/src/events/emit/index.ts
+++ b/packages/backend-core/src/events/emit/index.ts
@ -0,0 +1 @@
+export * from "./BBEventEmitter"
--- a/packages/backend-core/src/events/events.ts
+++ b/packages/backend-core/src/events/events.ts
@ -2,6 +2,41 @@ import { Event } from "@budibase/types"
 import { processors } from "./processors"
 import * as identification from "./identification"
 import * as backfill from "./backfill"
+import { emitter, EmittedEvent } from "./emit"
+import * as context from "../context"
+import * as logging from "../logging"
+
+const USE_EMITTER: any[] = [
+  Event.SERVED_BUILDER,
+  Event.SERVED_APP,
+  Event.SERVED_APP_PREVIEW,
+]
+
+for (let event of USE_EMITTER) {
+  emitter.on(event, async (props: EmittedEvent) => {
+    try {
+      await context.doInTenant(props.tenantId, async () => {
+        if (props.appId) {
+          await context.doInAppContext(props.appId, async () => {
+            await processors.processEvent(
+              event as Event,
+              props.identity,
+              props.properties
+            )
+          })
+        } else {
+          await processors.processEvent(
+            event as Event,
+            props.identity,
+            props.properties
+          )
+        }
+      })
+    } catch (e) {
+      logging.logAlert(`Unable to process async event ${event}`, e)
+    }
+  })
+}

 export const publishEvent = async (
  event: Event,
@ -11,6 +46,11 @@ export const publishEvent = async (
  // in future this should use async events via a distributed queue.
  const identity = await identification.getCurrentIdentity()

+  if (USE_EMITTER.includes(event)) {
+    emitter.emitEvent(event, properties, identity)
+    return
+  }
+
  const backfilling = await backfill.isBackfillingEvent(event)
  // no backfill - send the event and exit
  if (!backfilling) {
--- a/packages/backend-core/src/events/processors/AnalyticsProcessor.ts
+++ b/packages/backend-core/src/events/processors/AnalyticsProcessor.ts
@ -32,7 +32,7 @@ export default class AnalyticsProcessor implements EventProcessor {
      return
    }
    if (this.posthog) {
-      this.posthog.processEvent(event, identity, properties, timestamp)
+      await this.posthog.processEvent(event, identity, properties, timestamp)
    }
  }

@ -45,14 +45,14 @@ export default class AnalyticsProcessor implements EventProcessor {
      return
    }
    if (this.posthog) {
-      this.posthog.identify(identity, timestamp)
+      await this.posthog.identify(identity, timestamp)
    }
  }

  async identifyGroup(group: Group, timestamp?: string | number) {
    // Group indentifications (tenant and installation) always on
    if (this.posthog) {
-      this.posthog.identifyGroup(group, timestamp)
+      await this.posthog.identifyGroup(group, timestamp)
    }
  }

--- a/packages/backend-core/src/events/processors/posthog/rateLimiting.ts
+++ b/packages/backend-core/src/events/processors/posthog/rateLimiting.ts
@ -45,7 +45,7 @@ export const limited = async (event: Event): Promise<boolean> => {
    return false
  }

-  const cachedEvent = (await readEvent(event)) as EventProperties
+  const cachedEvent = await readEvent(event)
  if (cachedEvent) {
    const timestamp = new Date(cachedEvent.timestamp)
    const limit = RATE_LIMITS[event]
@ -76,14 +76,17 @@ export const limited = async (event: Event): Promise<boolean> => {
 const eventKey = (event: RateLimitedEvent) => {
  let key = `${CacheKeys.EVENTS_RATE_LIMIT}:${event}`
  if (isPerApp(event)) {
-    key = key + context.getAppId()
+    key = key + ":" + context.getAppId()
  }
  return key
 }

-const readEvent = async (event: RateLimitedEvent) => {
+const readEvent = async (
+  event: RateLimitedEvent
+): Promise<EventProperties | undefined> => {
  const key = eventKey(event)
-  return cache.get(key)
+  const result = await cache.get(key)
+  return result as EventProperties
 }

 const recordEvent = async (
--- a/packages/bbui/package.json
+++ b/packages/bbui/package.json
@ -1,7 +1,7 @@
 {
  "name": "@budibase/bbui",
  "description": "A UI solution used in the different Budibase projects.",
-  "version": "1.2.26-alpha.0",
+  "version": "1.2.27",
  "license": "MPL-2.0",
  "svelte": "src/index.js",
  "module": "dist/bbui.es.js",
@ -38,7 +38,7 @@
  ],
  "dependencies": {
    "@adobe/spectrum-css-workflow-icons": "^1.2.1",
-    "@budibase/string-templates": "1.2.26-alpha.0",
+    "@budibase/string-templates": "^1.2.27",
    "@spectrum-css/actionbutton": "^1.0.1",
    "@spectrum-css/actiongroup": "^1.0.1",
    "@spectrum-css/avatar": "^3.0.2",
--- a/packages/builder/package.json
+++ b/packages/builder/package.json
@ -1,6 +1,6 @@
 {
  "name": "@budibase/builder",
-  "version": "1.2.26-alpha.0",
+  "version": "1.2.27",
  "license": "GPL-3.0",
  "private": true,
  "scripts": {
@ -69,10 +69,10 @@
    }
  },
  "dependencies": {
-    "@budibase/bbui": "1.2.26-alpha.0",
-    "@budibase/client": "1.2.26-alpha.0",
-    "@budibase/frontend-core": "1.2.26-alpha.0",
-    "@budibase/string-templates": "1.2.26-alpha.0",
+    "@budibase/bbui": "^1.2.27",
+    "@budibase/client": "^1.2.27",
+    "@budibase/frontend-core": "^1.2.27",
+    "@budibase/string-templates": "^1.2.27",
    "@sentry/browser": "5.19.1",
    "@spectrum-css/page": "^3.0.1",
    "@spectrum-css/vars": "^3.0.1",
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@ -1,6 +1,6 @@
 {
  "name": "@budibase/cli",
-  "version": "1.2.26-alpha.0",
+  "version": "1.2.27",
  "description": "Budibase CLI, for developers, self hosting and migrations.",
  "main": "src/index.js",
  "bin": {
--- a/packages/client/package.json
+++ b/packages/client/package.json
@ -1,6 +1,6 @@
 {
  "name": "@budibase/client",
-  "version": "1.2.26-alpha.0",
+  "version": "1.2.27",
  "license": "MPL-2.0",
  "module": "dist/budibase-client.js",
  "main": "dist/budibase-client.js",
@ -19,9 +19,9 @@
    "dev:builder": "rollup -cw"
  },
  "dependencies": {
-    "@budibase/bbui": "1.2.26-alpha.0",
-    "@budibase/frontend-core": "1.2.26-alpha.0",
-    "@budibase/string-templates": "1.2.26-alpha.0",
+    "@budibase/bbui": "^1.2.27",
+    "@budibase/frontend-core": "^1.2.27",
+    "@budibase/string-templates": "^1.2.27",
    "@spectrum-css/button": "^3.0.3",
    "@spectrum-css/card": "^3.0.3",
    "@spectrum-css/divider": "^1.0.3",
--- a/packages/frontend-core/package.json
+++ b/packages/frontend-core/package.json
@ -1,12 +1,12 @@
 {
  "name": "@budibase/frontend-core",
-  "version": "1.2.26-alpha.0",
+  "version": "1.2.27",
  "description": "Budibase frontend core libraries used in builder and client",
  "author": "Budibase",
  "license": "MPL-2.0",
  "svelte": "src/index.js",
  "dependencies": {
-    "@budibase/bbui": "1.2.26-alpha.0",
+    "@budibase/bbui": "^1.2.27",
    "lodash": "^4.17.21",
    "svelte": "^3.46.2"
  }
--- a/packages/server/package.json
+++ b/packages/server/package.json
@ -1,7 +1,7 @@
 {
  "name": "@budibase/server",
  "email": "hi@budibase.com",
-  "version": "1.2.26-alpha.0",
+  "version": "1.2.27",
  "description": "Budibase Web Server",
  "main": "src/index.ts",
  "repository": {
@ -77,11 +77,11 @@
  "license": "GPL-3.0",
  "dependencies": {
    "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "1.2.26-alpha.0",
-    "@budibase/client": "1.2.26-alpha.0",
-    "@budibase/pro": "1.2.26-alpha.0",
-    "@budibase/string-templates": "1.2.26-alpha.0",
-    "@budibase/types": "1.2.26-alpha.0",
+    "@budibase/backend-core": "^1.2.27",
+    "@budibase/client": "^1.2.27",
+    "@budibase/pro": "1.2.27",
+    "@budibase/string-templates": "^1.2.27",
+    "@budibase/types": "^1.2.27",
    "@bull-board/api": "3.7.0",
    "@bull-board/koa": "3.9.4",
    "@elastic/elasticsearch": "7.10.0",
--- a/packages/server/src/api/controllers/static/index.ts
+++ b/packages/server/src/api/controllers/static/index.ts
@ -18,14 +18,14 @@ const { DocumentTypes, isDevAppID } = require("../../../db/utils")
 const { getAppDB, getAppId } = require("@budibase/backend-core/context")
 const { setCookie, clearCookie } = require("@budibase/backend-core/utils")
 const AWS = require("aws-sdk")
-const { events } = require("@budibase/backend-core")
+import { events } from "@budibase/backend-core"

 const fs = require("fs")
 const {
  downloadTarballDirect,
 } = require("../../../utilities/fileSystem/utilities")

-async function prepareUpload({ s3Key, bucket, metadata, file }) {
+async function prepareUpload({ s3Key, bucket, metadata, file }: any) {
  const response = await upload({
    bucket,
    metadata,
@ -44,7 +44,7 @@ async function prepareUpload({ s3Key, bucket, metadata, file }) {
  }
 }

-exports.toggleBetaUiFeature = async function (ctx) {
+export const toggleBetaUiFeature = async function (ctx: any) {
  const cookieName = `beta:${ctx.params.feature}`

  if (ctx.cookies.get(cookieName)) {
@ -72,21 +72,21 @@ exports.toggleBetaUiFeature = async function (ctx) {
  }
 }

-exports.serveBuilder = async function (ctx) {
+export const serveBuilder = async function (ctx: any) {
  const builderPath = resolve(TOP_LEVEL_PATH, "builder")
  await send(ctx, ctx.file, { root: builderPath })
-  if (!ctx.file.includes("assets/")) {
+  if (ctx.file === "index.html") {
    await events.serve.servedBuilder()
  }
 }

-exports.uploadFile = async function (ctx) {
+export const uploadFile = async function (ctx: any) {
  let files =
    ctx.request.files.file.length > 1
      ? Array.from(ctx.request.files.file)
      : [ctx.request.files.file]

-  const uploads = files.map(async file => {
+  const uploads = files.map(async (file: any) => {
    const fileExtension = [...file.name.split(".")].pop()
    // filenames converted to UUIDs so they are unique
    const processedFileName = `${uuid.v4()}.${fileExtension}`
@ -101,7 +101,7 @@ exports.uploadFile = async function (ctx) {
  ctx.body = await Promise.all(uploads)
 }

-exports.serveApp = async function (ctx) {
+export const serveApp = async function (ctx: any) {
  const db = getAppDB({ skip_setup: true })
  const appInfo = await db.get(DocumentTypes.APP_METADATA)
  let appId = getAppId()
@ -134,13 +134,13 @@ exports.serveApp = async function (ctx) {
  }
 }

-exports.serveClientLibrary = async function (ctx) {
+export const serveClientLibrary = async function (ctx: any) {
  return send(ctx, "budibase-client.js", {
    root: join(NODE_MODULES_PATH, "@budibase", "client", "dist"),
  })
 }

-exports.getSignedUploadURL = async function (ctx) {
+export const getSignedUploadURL = async function (ctx: any) {
  const database = getAppDB()

  // Ensure datasource is valid
--- a/packages/string-templates/package.json
+++ b/packages/string-templates/package.json
@ -1,6 +1,6 @@
 {
  "name": "@budibase/string-templates",
-  "version": "1.2.26-alpha.0",
+  "version": "1.2.27",
  "description": "Handlebars wrapper for Budibase templating.",
  "main": "src/index.cjs",
  "module": "dist/bundle.mjs",
--- a/packages/types/package.json
+++ b/packages/types/package.json
@ -1,6 +1,6 @@
 {
  "name": "@budibase/types",
-  "version": "1.2.26-alpha.0",
+  "version": "1.2.27",
  "description": "Budibase types",
  "main": "dist/index.js",
  "types": "dist/index.d.ts",
--- a/packages/worker/package.json
+++ b/packages/worker/package.json
@ -1,7 +1,7 @@
 {
  "name": "@budibase/worker",
  "email": "hi@budibase.com",
-  "version": "1.2.26-alpha.0",
+  "version": "1.2.27",
  "description": "Budibase background service",
  "main": "src/index.ts",
  "repository": {
@ -35,10 +35,10 @@
  "author": "Budibase",
  "license": "GPL-3.0",
  "dependencies": {
-    "@budibase/backend-core": "1.2.26-alpha.0",
-    "@budibase/pro": "1.2.26-alpha.0",
-    "@budibase/string-templates": "1.2.26-alpha.0",
-    "@budibase/types": "1.2.26-alpha.0",
+    "@budibase/backend-core": "^1.2.27",
+    "@budibase/pro": "1.2.27",
+    "@budibase/string-templates": "^1.2.27",
+    "@budibase/types": "^1.2.27",
    "@koa/router": "8.0.8",
    "@sentry/node": "6.17.7",
    "@techpass/passport-openidconnect": "0.3.2",