cURL import working

5 years ago · c421499a46
5 changed files with 174 additions and 9 deletions
--- a/packages/builder/src/components/backend/DatasourceNavigator/modals/ImportRestQueriesModal.svelte
+++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/ImportRestQueriesModal.svelte
@ -56,8 +56,7 @@
        datasourceId,
      }

-      const resp = await queries.import(body)
-      datasourceId = resp.datasourceId
+      await queries.import(body)

      // reload
      await datasources.fetch()
--- a/packages/builder/src/stores/backend/queries.js
+++ b/packages/builder/src/stores/backend/queries.js
@ -55,6 +55,11 @@ export function createQueriesStore() {
    },
    import: async body => {
      const response = await api.post(`/api/queries/import`, body)
+
+      if (response.status !== 200) {
+        throw new Error(response.message)
+      }
+
      return response.json()
    },
    select: query => {
--- a/packages/server/package.json
+++ b/packages/server/package.json
@ -85,6 +85,7 @@
    "bull": "^3.22.4",
    "chmodr": "1.2.0",
    "csvtojson": "2.0.10",
+    "curlconverter": "^3.21.0",
    "dotenv": "8.2.0",
    "download": "8.0.0",
    "fix-path": "3.0.0",
--- a/packages/server/src/api/controllers/query/import.ts
+++ b/packages/server/src/api/controllers/query/import.ts
@ -2,6 +2,8 @@ import CouchDB from "../../../db"
 import { queryValidation } from "./validation"
 import { generateQueryID } from "../../../db/utils"
 import { Spec as Swagger2, Operation } from "swagger-schema-official"
+const curlconverter = require("curlconverter")
+import { URL } from "url"

 // {
 //   "_id": "query_datasource_d62738f2d72a466997ffbf46f4952404_e7258ad382cd4c37961b81730633ff2d",
@ -90,15 +92,45 @@ interface DatasourceInfo {
 }

 const parseImportStrategy = (data: string): Strategy => {
-  return Strategy.SWAGGER2
+  try {
+    const json = JSON.parse(data)
+    if (json.swagger === "2.0") {
+      return Strategy.CURL
+    } else if (json.openapi?.includes("3.0")) {
+      return Strategy.OPENAPI3
+    }
+  } catch (jsonError) {
+    try {
+      parseCurl(data)
+      return Strategy.CURL
+    } catch (curlError) {
+      // do nothing
+    }
+  }
+
+  throw new Error(`The import data could not be processed`)
+}
+
+const processPath = (path: string): string => {
+  if (path?.startsWith("/")) {
+    return path.substring(1)
+  }
+
+  return path
 }

 // SWAGGER

 const parseSwagger2Info = (swagger2: Swagger2): DatasourceInfo => {
+  const scheme = swagger2.schemes?.includes("https") ? "https" : "http"
+  const basePath = swagger2.basePath || ""
+  const host = swagger2.host || "<host>"
+  const url = `${scheme}://${host}${basePath}`
+  const name = swagger2.info.title || "Swagger Import"
+
  return {
-    url: "http://localhost:3000",
-    name: "swagger",
+    url: url,
+    name: name,
    defaultHeaders: [],
  }
 }
@ -150,16 +182,42 @@ const parseOpenAPI3Queries = (datasourceId: string, data: string): Query[] => {

 // CURL

+const parseCurl = (data: string): any => {
+  const curlJson = curlconverter.toJsonString(data)
+  return JSON.parse(curlJson)
+}
+
 const parseCurlDatasourceInfo = (data: any): DatasourceInfo => {
+  const curl = parseCurl(data)
+
+  const url = new URL(curl.url)
+
  return {
-    url: "http://localhost:3000",
-    name: "swagger",
+    url: url.origin,
+    name: url.hostname,
    defaultHeaders: [],
  }
 }

 const parseCurlQueries = (datasourceId: string, data: string): Query[] => {
-  return []
+  const curl = parseCurl(data)
+
+  const url = new URL(curl.url)
+  const name = url.pathname
+  const path = url.pathname
+  const method = curl.method
+  const queryString = url.search
+  const headers = curl.headers
+
+  const query = constructQuery(
+    datasourceId,
+    name,
+    method,
+    path,
+    queryString,
+    headers
+  )
+  return [query]
 }

 const verbFromMethod = (method: string) => {
@ -183,6 +241,7 @@ const constructQuery = (
  const queryVerb = verbFromMethod(method)
  const transformer = "return data"
  const schema = {}
+  path = processPath(path)

  const query: Query = {
    datasourceId,
--- a/packages/server/yarn.lock
+++ b/packages/server/yarn.lock
@ -1190,6 +1190,24 @@
  dependencies:
    "@cspotcode/source-map-consumer" "0.8.0"

+"@curlconverter/yargs-parser@^0.0.1":
+  version "0.0.1"
+  resolved "https://registry.yarnpkg.com/@curlconverter/yargs-parser/-/yargs-parser-0.0.1.tgz#62a360cea3d62b9b5805e61e8110cea98da8d140"
+  integrity sha512-DbEVRYqrorzwqc63MQ3RODflut1tNla8ZCKo1h83lF7+fbntgubZsDfRDBv5Lxj3vkKuvAolysNM2ekwJev8wA==
+
+"@curlconverter/yargs@^0.0.2":
+  version "0.0.2"
+  resolved "https://registry.yarnpkg.com/@curlconverter/yargs/-/yargs-0.0.2.tgz#a8ad4a2a2d0b0f897f8c117f664199165b59fd8a"
+  integrity sha512-Q1YEebpCY61kxme4wvU0/IN/uMBfG5pZOKCo9FU+w20ElPvN+eH2qEVbK1C12t3Tee3qeYLLEU6HkiUeO1gc4A==
+  dependencies:
+    "@curlconverter/yargs-parser" "^0.0.1"
+    cliui "^7.0.2"
+    escalade "^3.1.1"
+    get-caller-file "^2.0.5"
+    require-directory "^2.1.1"
+    string-width "^4.2.0"
+    y18n "^5.0.5"
+
 "@elastic/elasticsearch@7.10.0":
  version "7.10.0"
  resolved "https://registry.yarnpkg.com/@elastic/elasticsearch/-/elasticsearch-7.10.0.tgz#da105a9c1f14146f9f2cab4e7026cb7949121b8d"
@ -2562,6 +2580,11 @@
  resolved "https://registry.yarnpkg.com/@xmldom/xmldom/-/xmldom-0.7.5.tgz#09fa51e356d07d0be200642b0e4f91d8e6dd408d"
  integrity sha512-V3BIhmY36fXZ1OtVcI9W+FxQqxVLsPKcNjWigIaa81dLC9IolJl5Mt4Cvhmr0flUnjSpTdrbMTSbXqYqV5dT6A==

+a-sync-waterfall@^1.0.0:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/a-sync-waterfall/-/a-sync-waterfall-1.0.1.tgz#75b6b6aa72598b497a125e7a2770f14f4c8a1fa7"
+  integrity sha512-RYTOHHdWipFUliRFMCS4X2Yn2X8M87V/OpSqWzKKOGhzqyUxzyVmhHDH9sAvG+ZuQf/TAOFsLCpMw09I1ufUnA==
+
 abab@^2.0.0, abab@^2.0.3, abab@^2.0.5:
  version "2.0.5"
  resolved "https://registry.yarnpkg.com/abab/-/abab-2.0.5.tgz#c0b678fb32d60fc1219c784d6a826fe385aeb79a"
@ -2896,6 +2919,11 @@ array-unique@^0.3.2:
  resolved "https://registry.yarnpkg.com/array-unique/-/array-unique-0.3.2.tgz#a894b75d4bc4f6cd679ef3244a9fd8f46ae2d428"
  integrity sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg=

+asap@^2.0.3:
+  version "2.0.6"
+  resolved "https://registry.yarnpkg.com/asap/-/asap-2.0.6.tgz#e50347611d7e690943208bbdafebcbc2fb866d46"
+  integrity sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY=
+
 asn1@~0.2.3:
  version "0.2.6"
  resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.6.tgz#0d3a7bb6e64e02a90c0303b31f292868ea09a08d"
@ -3764,6 +3792,11 @@ commander@^2.19.0, commander@^2.5.0, commander@^2.8.1:
  resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33"
  integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==

+commander@^5.1.0:
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-5.1.0.tgz#46abbd1652f8e059bddaef99bbdcb2ad9cf179ae"
+  integrity sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==
+
 commander@^7.1.0:
  version "7.2.0"
  resolved "https://registry.yarnpkg.com/commander/-/commander-7.2.0.tgz#a36cb57d0b501ce108e4d20559a150a391d97ab7"
@ -4016,6 +4049,19 @@ csvtojson@2.0.10:
    lodash "^4.17.3"
    strip-bom "^2.0.0"

+curlconverter@^3.21.0:
+  version "3.21.0"
+  resolved "https://registry.yarnpkg.com/curlconverter/-/curlconverter-3.21.0.tgz#50b9568b8d24967ef55a0fcd872e7a18eea4cbc2"
+  integrity sha512-DXCnp1A/Xa69FujksUfdvWQFAnIn/C+4Wuv8t+UVdZkF/lY5bzj98GGKOGme7V/ckSHDLxE29Xp76sJ5Cpsp5A==
+  dependencies:
+    "@curlconverter/yargs" "^0.0.2"
+    cookie "^0.4.1"
+    jsesc "^3.0.2"
+    nunjucks "^3.2.3"
+    query-string "^7.0.1"
+    string.prototype.startswith "^1.0.0"
+    yamljs "^0.3.0"
+
 dashdash@^1.12.0:
  version "1.14.1"
  resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0"
@ -4537,7 +4583,7 @@ error-inject@^1.0.0:
  resolved "https://registry.yarnpkg.com/error-inject/-/error-inject-1.0.0.tgz#e2b3d91b54aed672f309d950d154850fa11d4f37"
  integrity sha1-4rPZG1Su1nLzCdlQ0VSFD6EdTzc=

-es-abstract@^1.19.1:
+es-abstract@^1.17.5, es-abstract@^1.19.1:
  version "1.19.1"
  resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.19.1.tgz#d4885796876916959de78edaa0df456627115ec3"
  integrity sha512-2vJ6tjA/UfqLm2MPs7jxVybLoB8i1t1Jd9R3kISld20sIxPcTbLuggQOUxeWeAvIUkduv/CfMjuh4WmiXr2v9w==
@ -5147,6 +5193,11 @@ fill-range@^7.0.1:
  dependencies:
    to-regex-range "^5.0.1"

+filter-obj@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/filter-obj/-/filter-obj-1.1.0.tgz#9b311112bc6c6127a16e016c6c5d7f19e0805c5b"
+  integrity sha1-mzERErxsYSehbgFsbF1/GeCAXFs=
+
 find-up@^3.0.0:
  version "3.0.0"
  resolved "https://registry.yarnpkg.com/find-up/-/find-up-3.0.0.tgz#49169f1d7993430646da61ecc5ae355c21c97b73"
@ -7447,6 +7498,11 @@ jsesc@^2.5.1:
  resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4"
  integrity sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==

+jsesc@^3.0.2:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-3.0.2.tgz#bb8b09a6597ba426425f2e4a07245c3d00b9343e"
+  integrity sha512-xKqzzWXDttJuOcawBt4KnKHHIf5oQ/Cxax+0PWFG+DFDgHNAdi+TXECADI+RYiFUMmx8792xsMbbgXj4CwnP4g==
+
 jsesc@~0.5.0:
  version "0.5.0"
  resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-0.5.0.tgz#e7dee66e35d6fc16f710fe91d5cf69f70f08911d"
@ -8732,6 +8788,15 @@ npm-run-path@^4.0.1:
  dependencies:
    path-key "^3.0.0"

+nunjucks@^3.2.3:
+  version "3.2.3"
+  resolved "https://registry.yarnpkg.com/nunjucks/-/nunjucks-3.2.3.tgz#1b33615247290e94e28263b5d855ece765648a31"
+  integrity sha512-psb6xjLj47+fE76JdZwskvwG4MYsQKXUtMsPh6U0YMvmyjRtKRFcxnlXGWglNybtNTNVmGdp94K62/+NjF5FDQ==
+  dependencies:
+    a-sync-waterfall "^1.0.0"
+    asap "^2.0.3"
+    commander "^5.1.0"
+
 nwsapi@^2.0.7, nwsapi@^2.2.0:
  version "2.2.0"
  resolved "https://registry.yarnpkg.com/nwsapi/-/nwsapi-2.2.0.tgz#204879a9e3d068ff2a55139c2c772780681a38b7"
@ -9795,6 +9860,16 @@ query-string@^5.0.1:
    object-assign "^4.1.0"
    strict-uri-encode "^1.0.0"

+query-string@^7.0.1:
+  version "7.0.1"
+  resolved "https://registry.yarnpkg.com/query-string/-/query-string-7.0.1.tgz#45bd149cf586aaa582dffc7ec7a8ad97dd02f75d"
+  integrity sha512-uIw3iRvHnk9to1blJCG3BTc+Ro56CBowJXKmNNAm3RulvPBzWLRqKSiiDk+IplJhsydwtuNMHi8UGQFcCLVfkA==
+  dependencies:
+    decode-uri-component "^0.2.0"
+    filter-obj "^1.1.0"
+    split-on-first "^1.0.0"
+    strict-uri-encode "^2.0.0"
+
 querystring@0.2.0:
  version "0.2.0"
  resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620"
@ -10677,6 +10752,11 @@ spdx-license-ids@^3.0.0:
  resolved "https://registry.yarnpkg.com/spdx-license-ids/-/spdx-license-ids-3.0.11.tgz#50c0d8c40a14ec1bf449bae69a0ea4685a9d9f95"
  integrity sha512-Ctl2BrFiM0X3MANYgj3CkygxhRmr9mi6xhejbdO960nF6EDJApTYpn0BQnDKlnNBULKiCN1n3w9EBkHK8ZWg+g==

+split-on-first@^1.0.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/split-on-first/-/split-on-first-1.1.0.tgz#f610afeee3b12bce1d0c30425e76398b78249a5f"
+  integrity sha512-43ZssAJaMusuKWL8sKUBQXHWOpq8d6CfN/u1p4gUzfJkM05C8rxTmYrkIPTXapZpORA6LkkzcUulJ8FqA7Uudw==
+
 split-string@^3.0.1, split-string@^3.0.2:
  version "3.1.0"
  resolved "https://registry.yarnpkg.com/split-string/-/split-string-3.1.0.tgz#7cb09dda3a86585705c64b39a6466038682e8fe2"
@ -10785,6 +10865,11 @@ strict-uri-encode@^1.0.0:
  resolved "https://registry.yarnpkg.com/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz#279b225df1d582b1f54e65addd4352e18faa0713"
  integrity sha1-J5siXfHVgrH1TmWt3UNS4Y+qBxM=

+strict-uri-encode@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz#b9c7330c7042862f6b142dc274bbcc5866ce3546"
+  integrity sha1-ucczDHBChi9rFC3CdLvMWGbONUY=
+
 string-length@^4.0.1:
  version "4.0.2"
  resolved "https://registry.yarnpkg.com/string-length/-/string-length-4.0.2.tgz#a8a8dc7bd5c1a82b9b3c8b87e125f66871b6e57a"
@ -10816,6 +10901,14 @@ string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2
    is-fullwidth-code-point "^3.0.0"
    strip-ansi "^6.0.1"

+string.prototype.startswith@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/string.prototype.startswith/-/string.prototype.startswith-1.0.0.tgz#92a361fb1ac172033d53eb1db3d659b0cfab6280"
+  integrity sha512-VHhsDkuf8gsw4JNRK9cIZjYe6r7PsVUutVohaBhqYAoPaRADoQH+mMgUg7Cs/TgQeDGEvI+PzPEMOdvdsCMvpg==
+  dependencies:
+    define-properties "^1.1.3"
+    es-abstract "^1.17.5"
+
 string.prototype.trimend@^1.0.4:
  version "1.0.4"
  resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.4.tgz#e75ae90c2942c63504686c18b287b4a0b1a45f80"
@ -12137,6 +12230,14 @@ yaml@^1.10.2:
  resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b"
  integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==

+yamljs@^0.3.0:
+  version "0.3.0"
+  resolved "https://registry.yarnpkg.com/yamljs/-/yamljs-0.3.0.tgz#dc060bf267447b39f7304e9b2bfbe8b5a7ddb03b"
+  integrity sha512-C/FsVVhht4iPQYXOInoxUM/1ELSf9EsgKH34FofQOp6hwCPrW4vG4w5++TED3xRUo8gD7l0P1J1dLlDYzODsTQ==
+  dependencies:
+    argparse "^1.0.7"
+    glob "^7.0.5"
+
 yargs-parser@20.x, yargs-parser@^20.2.2:
  version "20.2.9"
  resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.9.tgz#2eb7dc3b0289718fc295f362753845c41a0c94ee"