From b57b0f1c7b7ebbcdb596866734fe9fa1bc591775 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 29 Dec 2021 15:46:33 +0000 Subject: [PATCH 1/3] adding dynamoDB describe call to dynamo integration --- packages/server/src/integrations/dynamodb.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/packages/server/src/integrations/dynamodb.ts b/packages/server/src/integrations/dynamodb.ts index 0bc3a1273..a3f32e5bf 100644 --- a/packages/server/src/integrations/dynamodb.ts +++ b/packages/server/src/integrations/dynamodb.ts @@ -80,6 +80,17 @@ module DynamoModule { }, }, }, + describe: { + type: QueryTypes.FIELDS, + customisable: true, + readable: true, + fields: { + table: { + type: DatasourceFieldTypes.STRING, + required: true, + }, + }, + }, get: { type: QueryTypes.FIELDS, customisable: true, @@ -180,6 +191,13 @@ module DynamoModule { return response } + async describe(query: { table: string; }) { + const params = { + TableName: query.table, + } + return new AWS.DynamoDB().describeTable(params).promise() + } + async get(query: { table: string; json: object }) { const params = { TableName: query.table, From 18ad8ac1e5fc62083041a3153445995f7581e2bd Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 30 Dec 2021 17:44:27 +0000 Subject: [PATCH 2/3] configurable query timeout, s3 options for minIO, safety on cloud deploy job --- .github/workflows/deploy-cloud.yaml | 6 ++++++ packages/server/src/api/controllers/query.js | 5 ++++- packages/server/src/environment.js | 1 + packages/server/src/integrations/dynamodb.ts | 2 +- packages/server/src/integrations/oracle.ts | 2 +- packages/server/src/integrations/s3.ts | 12 ++++++++++++ 6 files changed, 25 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy-cloud.yaml b/.github/workflows/deploy-cloud.yaml index 26422a2e7..d54e6c9c6 100644 --- a/.github/workflows/deploy-cloud.yaml +++ b/.github/workflows/deploy-cloud.yaml @@ -12,6 +12,12 @@ jobs: runs-on: ubuntu-latest steps: + - name: Fail if branch is not master + if: github.ref != 'refs/heads/master' + run: | + echo "Ref is not master, you must run this job from master." + exit 1 + - uses: actions/checkout@v2 - name: Pull values.yaml from budibase-infra diff --git a/packages/server/src/api/controllers/query.js b/packages/server/src/api/controllers/query.js index cf6f03f00..4610fe57c 100644 --- a/packages/server/src/api/controllers/query.js +++ b/packages/server/src/api/controllers/query.js @@ -7,8 +7,11 @@ const { } = require("../../db/utils") const { BaseQueryVerbs } = require("../../constants") const { Thread, ThreadType } = require("../../threads") +const env = require("../../environment") -const Runner = new Thread(ThreadType.QUERY, { timeoutMs: 10000 }) +const Runner = new Thread(ThreadType.QUERY, { + timeoutMs: env.QUERY_THREAD_TIMEOUT || 10000, +}) // simple function to append "readable" to all read queries function enrichQueries(input) { diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js index 925fede08..a92e11385 100644 --- a/packages/server/src/environment.js +++ b/packages/server/src/environment.js @@ -65,6 +65,7 @@ module.exports = { DEPLOYMENT_CREDENTIALS_URL: process.env.DEPLOYMENT_CREDENTIALS_URL, ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS, DISABLE_THREADING: process.env.DISABLE_THREADING, + QUERY_THREAD_TIMEOUT: process.env.QUERY_THREAD_TIMEOUT, _set(key, value) { process.env[key] = value module.exports[key] = value diff --git a/packages/server/src/integrations/dynamodb.ts b/packages/server/src/integrations/dynamodb.ts index a3f32e5bf..bfcac3184 100644 --- a/packages/server/src/integrations/dynamodb.ts +++ b/packages/server/src/integrations/dynamodb.ts @@ -191,7 +191,7 @@ module DynamoModule { return response } - async describe(query: { table: string; }) { + async describe(query: { table: string }) { const params = { TableName: query.table, } diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index afaa90265..f602a9759 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -450,7 +450,7 @@ module OracleModule { }) return lastRow.rows } else { - return [{ [ operation.toLowerCase() ]: true }] + return [{ [operation.toLowerCase()]: true }] } } } diff --git a/packages/server/src/integrations/s3.ts b/packages/server/src/integrations/s3.ts index bcbcdf342..b258670ee 100644 --- a/packages/server/src/integrations/s3.ts +++ b/packages/server/src/integrations/s3.ts @@ -29,6 +29,18 @@ module S3Module { type: "password", required: true, }, + endpoint: { + type: "string", + required: false, + }, + s3ForcePathStyle: { + type: "boolean", + required: false, + }, + signatureVersion: { + type: "string", + required: false, + }, }, query: { read: { From e87a348fb885b2851223e16404664bd10e6c657e Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Fri, 31 Dec 2021 16:15:49 +0000 Subject: [PATCH 3/3] enabling minio support for S3 connector --- packages/server/src/integrations/s3.ts | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/packages/server/src/integrations/s3.ts b/packages/server/src/integrations/s3.ts index b258670ee..25b439fd5 100644 --- a/packages/server/src/integrations/s3.ts +++ b/packages/server/src/integrations/s3.ts @@ -8,6 +8,8 @@ module S3Module { region: string accessKeyId: string secretAccessKey: string + s3ForcePathStyle: boolean + endpoint?: string } const SCHEMA: Integration = { @@ -18,7 +20,7 @@ module S3Module { datasource: { region: { type: "string", - required: true, + required: false, default: "us-east-1", }, accessKeyId: { @@ -33,13 +35,10 @@ module S3Module { type: "string", required: false, }, - s3ForcePathStyle: { - type: "boolean", - required: false, - }, signatureVersion: { type: "string", required: false, + default: "v4" }, }, query: { @@ -58,16 +57,16 @@ module S3Module { class S3Integration implements IntegrationBase { private readonly config: S3Config private client: any - private connectionPromise: Promise constructor(config: S3Config) { this.config = config - this.connectionPromise = this.connect() - this.client = new AWS.S3() - } + if (this.config.endpoint) { + this.config.s3ForcePathStyle = true + } else { + delete this.config.endpoint + } - async connect() { - AWS.config.update(this.config) + this.client = new AWS.S3(this.config) } async read(query: { bucket: string }) {