Merge pull request #2756 from Budibase/fix/sql-fixes

Allow newlines in Postgres JSON inputs

packages/server/scripts/integrations/pg-json/docker-compose.yml

@@ -0,0 +1,28 @@
+version: "3.8"
+services:
+  db:
+    container_name: postgres-json
+    image: postgres
+    restart: always
+    environment:
+      POSTGRES_USER: root
+      POSTGRES_PASSWORD: root
+      POSTGRES_DB: main
+    ports:
+      - "5432:5432"
+    volumes:
+      #- pg_data:/var/lib/postgresql/data/
+      - ./init.sql:/docker-entrypoint-initdb.d/init.sql
+
+  pgadmin:
+    container_name: pgadmin-json
+    image: dpage/pgadmin4
+    restart: always
+    environment:
+      PGADMIN_DEFAULT_EMAIL: root@root.com
+      PGADMIN_DEFAULT_PASSWORD: root
+    ports:
+      - "5050:80"
+
+#volumes:
+#  pg_data:

packages/server/scripts/integrations/pg-json/init.sql

@@ -0,0 +1,22 @@
+SELECT 'CREATE DATABASE main'
+WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'main')\gexec
+CREATE TABLE jsonTable (
+    id character varying(32),
+    data jsonb,
+    text text
+);
+
+INSERT INTO jsonTable (id, data) VALUES ('1', '{"id": 1, "age": 1, "name": "Mike", "newline": "this is text with a\n newline in it"}');
+
+CREATE VIEW jsonView AS SELECT
+	x.id,
+	x.age,
+	x.name,
+	x.newline
+FROM
+	jsonTable c,
+	LATERAL jsonb_to_record(c.data) x (id character varying(32),
+		age BIGINT,
+		name TEXT,
+		newline TEXT
+		);

packages/server/scripts/integrations/pg-json/reset.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+docker-compose down
+docker volume prune -f

packages/server/src/integrations/postgres.ts

@@ -13,6 +13,9 @@ module PostgresModule {
   const Sql = require("./base/sql")
   const { FieldTypes } = require("../constants")
   const { buildExternalTableId, convertType, copyExistingPropsOver } = require("./utils")
+  const { escapeDangerousCharacters } = require("../utilities")
+
+  const JSON_REGEX = /'{.*}'::json/s
 
   interface PostgresConfig {
     host: string
@@ -94,6 +97,17 @@ module PostgresModule {
   }
 
   async function internalQuery(client: any, query: SqlQuery) {
+    // need to handle a specific issue with json data types in postgres,
+    // new lines inside the JSON data will break it
+    if (query && query.sql) {
+      const matches = query.sql.match(JSON_REGEX)
+      if (matches && matches.length > 0) {
+        for (let match of matches) {
+          const escaped = escapeDangerousCharacters(match)
+          query.sql = query.sql.replace(match, escaped)
+        }
+      }
+    }
     try {
       return await client.query(query.sql, query.bindings || [])
     } catch (err) {

packages/server/src/utilities/index.js

@@ -106,3 +106,13 @@ exports.deleteEntityMetadata = async (appId, type, entityId) => {
     await db.remove(id, rev)
   }
 }
+
+exports.escapeDangerousCharacters = string => {
+  return string
+    .replace(/[\\]/g, "\\\\")
+    .replace(/[\b]/g, "\\b")
+    .replace(/[\f]/g, "\\f")
+    .replace(/[\n]/g, "\\n")
+    .replace(/[\r]/g, "\\r")
+    .replace(/[\t]/g, "\\t")
+}