diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index d1e373003..2a57d6f38 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -137,7 +137,7 @@ If you wish to delete all the apps created in development and reset the environm ### Backend -For the backend we run [Redis](https://redis.io/), [CouchDB](https://couchdb.apache.org/), [MinIO](https://min.io/) and [Envoy](https://www.envoyproxy.io/) in Docker compose. This means that to develop Budibase you will need Docker and Docker compose installed. The backend services are then ran separately as Node services with nodemon so that they can be debugged outside of Docker. +For the backend we run [Redis](https://redis.io/), [CouchDB](https://couchdb.apache.org/), [MinIO](https://min.io/) and [NGINX](https://www.nginx.com/) in Docker compose. This means that to develop Budibase you will need Docker and Docker compose installed. The backend services are then ran separately as Node services with nodemon so that they can be debugged outside of Docker. ### Data Storage diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 7e9511541..7c1b1f94b 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -14,7 +14,7 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: [self-hosted, linux] strategy: matrix: diff --git a/.github/workflows/deploy-preprod.yml b/.github/workflows/deploy-preprod.yml index 5b3282313..10fadc36c 100644 --- a/.github/workflows/deploy-preprod.yml +++ b/.github/workflows/deploy-preprod.yml @@ -38,7 +38,7 @@ jobs: wc -l values.preprod.yaml - name: Deploy to Preprod Environment - uses: deliverybot/helm@v1 + uses: glopezep/helm@v1.7.1 with: release: budibase-preprod namespace: budibase diff --git a/.gitignore b/.gitignore index 6ba2f61ed..d98e8e8fc 100644 --- a/.gitignore +++ b/.gitignore @@ -64,7 +64,7 @@ typings/ # dotenv environment variables file .env !hosting/.env -hosting/.generated-envoy.dev.yaml +hosting/.generated-nginx.dev.conf # parcel-bundler cache (https://parceljs.org/) .cache diff --git a/charts/budibase/Chart.yaml b/charts/budibase/Chart.yaml index 8c9d44f20..766657769 100644 --- a/charts/budibase/Chart.yaml +++ b/charts/budibase/Chart.yaml @@ -11,8 +11,8 @@ sources: - https://github.com/Budibase/budibase - https://budibase.com type: application -version: 0.2.5 -appVersion: 1.0.25 +version: 0.2.6 +appVersion: 1.0.48 dependencies: - name: couchdb version: 3.3.4 diff --git a/charts/budibase/templates/proxy-service-deployment.yaml b/charts/budibase/templates/proxy-service-deployment.yaml index 2e453d1c5..9ea7df160 100644 --- a/charts/budibase/templates/proxy-service-deployment.yaml +++ b/charts/budibase/templates/proxy-service-deployment.yaml @@ -25,7 +25,7 @@ spec: app.kubernetes.io/name: budibase-proxy spec: containers: - - image: budibase/proxy + - image: budibase/proxy:k8s imagePullPolicy: Always name: proxy-service ports: diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index 8b6f5564a..b6c757cb9 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -111,6 +111,10 @@ spec: value: {{ .Values.globals.smtp.from | quote }} - name: APPS_URL value: http://app-service:{{ .Values.services.apps.port }} + - name: GOOGLE_CLIENT_ID + value: {{ .Values.globals.google.clientId | quote }} + - name: GOOGLE_CLIENT_SECRET + value: {{ .Values.globals.google.secret | quote }} image: budibase/worker:{{ .Values.globals.appVersion }} imagePullPolicy: Always name: bbworker diff --git a/hosting/.env b/hosting/.env deleted file mode 120000 index bb1b54ad7..000000000 --- a/hosting/.env +++ /dev/null @@ -1 +0,0 @@ -hosting.properties \ No newline at end of file diff --git a/hosting/.env b/hosting/.env new file mode 100644 index 000000000..39df76d01 --- /dev/null +++ b/hosting/.env @@ -0,0 +1,21 @@ +# Use the main port in the builder for your self hosting URL, e.g. localhost:10000 +MAIN_PORT=10000 + +# This section contains all secrets pertaining to the system +# These should be updated +JWT_SECRET=testsecret +MINIO_ACCESS_KEY=budibase +MINIO_SECRET_KEY=budibase +COUCH_DB_PASSWORD=budibase +COUCH_DB_USER=budibase +REDIS_PASSWORD=budibase +INTERNAL_API_KEY=budibase + +# This section contains variables that do not need to be altered under normal circumstances +APP_PORT=4002 +WORKER_PORT=4003 +MINIO_PORT=4004 +COUCH_DB_PORT=4005 +REDIS_PORT=6379 +WATCHTOWER_PORT=6161 +BUDIBASE_ENVIRONMENT=PRODUCTION \ No newline at end of file diff --git a/hosting/digitalocean/files/var/lib/cloud/scripts/per-instance/001_onboot b/hosting/digitalocean/files/var/lib/cloud/scripts/per-instance/001_onboot index e5a883ac8..ffa63ad67 100755 --- a/hosting/digitalocean/files/var/lib/cloud/scripts/per-instance/001_onboot +++ b/hosting/digitalocean/files/var/lib/cloud/scripts/per-instance/001_onboot @@ -3,9 +3,8 @@ # go into the app dir cd /root -# fetch envoy and docker-compose files +# fetch nginx and docker-compose files wget https://raw.githubusercontent.com/Budibase/budibase/master/hosting/docker-compose.yaml -wget https://raw.githubusercontent.com/Budibase/budibase/master/hosting/envoy.yaml wget https://raw.githubusercontent.com/Budibase/budibase/master/hosting/hosting.properties # Create .env file from hosting.properties using bash and then remove it diff --git a/hosting/docker-compose.dev.yaml b/hosting/docker-compose.dev.yaml index eaced64e0..df403c0a2 100644 --- a/hosting/docker-compose.dev.yaml +++ b/hosting/docker-compose.dev.yaml @@ -22,18 +22,21 @@ services: retries: 3 proxy-service: - container_name: budi-envoy-dev + container_name: budi-nginx-dev restart: always - image: envoyproxy/envoy:v1.16-latest + image: nginx:latest volumes: - - ./.generated-envoy.dev.yaml:/etc/envoy/envoy.yaml + - ./.generated-nginx.dev.conf:/etc/nginx/nginx.conf ports: - "${MAIN_PORT}:10000" depends_on: - minio-service - couchdb-service + extra_hosts: + - "host.docker.internal:host-gateway" couchdb-service: + # platform: linux/amd64 container_name: budi-couchdb-dev restart: always image: ibmcom/couchdb3 diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index c94d1520a..17ed12a13 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -80,9 +80,8 @@ services: proxy-service: restart: always - image: envoyproxy/envoy:v1.16-latest - volumes: - - ./envoy.yaml:/etc/envoy/envoy.yaml + container_name: bbproxy + image: budibase/proxy ports: - "${MAIN_PORT}:10000" depends_on: @@ -125,7 +124,7 @@ services: - "${WATCHTOWER_PORT}:8080" volumes: - /var/run/docker.sock:/var/run/docker.sock - command: --debug --http-api-update bbapps bbworker + command: --debug --http-api-update bbapps bbworker bbproxy environment: - WATCHTOWER_HTTP_API=true - WATCHTOWER_HTTP_API_TOKEN=budibase diff --git a/hosting/envoy.dev.yaml.hbs b/hosting/envoy.dev.yaml.hbs deleted file mode 100644 index 59363fab5..000000000 --- a/hosting/envoy.dev.yaml.hbs +++ /dev/null @@ -1,149 +0,0 @@ -static_resources: - listeners: - - name: main_listener - address: - socket_address: { address: 0.0.0.0, port_value: 10000 } - filter_chains: - - filters: - - name: envoy.filters.network.http_connection_manager - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager - stat_prefix: ingress - codec_type: auto - route_config: - name: local_route - virtual_hosts: - - name: local_services - domains: ["*"] - routes: - # special case to redirect specifically the route path - # to the builder, if this were a prefix then it would break minio - - match: { path: "/" } - redirect: { path_redirect: "/builder/" } - - - match: { prefix: "/db/" } - route: - cluster: couchdb-service - prefix_rewrite: "/" - - - match: { prefix: "/api/system/" } - route: - cluster: worker-dev - - - match: { prefix: "/api/admin/" } - route: - cluster: worker-dev - - - match: { prefix: "/api/global/" } - route: - cluster: worker-dev - - - match: { prefix: "/api/" } - route: - cluster: server-dev - timeout: 120s - - - match: { prefix: "/app_" } - route: - cluster: server-dev - - - match: { prefix: "/app/" } - route: - cluster: server-dev - prefix_rewrite: "/" - - # the below three cases are needed to make sure - # all traffic prefixed for the builder is passed through - # correctly. - - match: { path: "/" } - route: - cluster: builder-dev - - - match: { prefix: "/builder/" } - route: - cluster: builder-dev - - - match: { prefix: "/builder" } - route: - cluster: builder-dev - prefix_rewrite: "/builder/" - - # minio is on the default route because this works - # best, minio + AWS SDK doesn't handle path proxy - - match: { prefix: "/" } - route: - cluster: minio-service - - http_filters: - - name: envoy.filters.http.router - - clusters: - - name: minio-service - connect_timeout: 0.25s - type: strict_dns - lb_policy: round_robin - load_assignment: - cluster_name: minio-service - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: minio-service - port_value: 9000 - - - name: couchdb-service - connect_timeout: 0.25s - type: strict_dns - lb_policy: round_robin - load_assignment: - cluster_name: couchdb-service - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: couchdb-service - port_value: 5984 - - - name: server-dev - connect_timeout: 0.25s - type: strict_dns - lb_policy: round_robin - load_assignment: - cluster_name: server-dev - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: {{ address }} - port_value: 4001 - - - name: builder-dev - connect_timeout: 15s - type: strict_dns - lb_policy: round_robin - load_assignment: - cluster_name: builder-dev - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: {{ address }} - port_value: 3000 - - - name: worker-dev - connect_timeout: 0.25s - type: strict_dns - lb_policy: round_robin - load_assignment: - cluster_name: worker-dev - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: {{ address }} - port_value: 4002 diff --git a/hosting/envoy.yaml b/hosting/envoy.yaml deleted file mode 100644 index d9f838468..000000000 --- a/hosting/envoy.yaml +++ /dev/null @@ -1,152 +0,0 @@ -static_resources: - listeners: - - name: main_listener - address: - socket_address: { address: 0.0.0.0, port_value: 10000 } - filter_chains: - - filters: - - name: envoy.filters.network.http_connection_manager - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager - stat_prefix: ingress - codec_type: auto - route_config: - name: local_route - virtual_hosts: - - name: local_services - domains: ["*"] - routes: - - match: { prefix: "/app/" } - route: - cluster: app-service - prefix_rewrite: "/" - - - match: { path: "/v1/update" } - route: - cluster: watchtower-service - - - match: { prefix: "/builder/" } - route: - cluster: app-service - - - match: { prefix: "/builder" } - route: - cluster: app-service - - - match: { prefix: "/app_" } - route: - cluster: app-service - - # special cases for worker admin (deprecated), global and system API - - match: { prefix: "/api/global/" } - route: - cluster: worker-service - - - match: { prefix: "/api/admin/" } - route: - cluster: worker-service - - - match: { prefix: "/api/system/" } - route: - cluster: worker-service - - - match: { path: "/" } - route: - cluster: app-service - - # special case for when API requests are made, can just forward, not to minio - - match: { prefix: "/api/" } - route: - cluster: app-service - timeout: 120s - - - match: { prefix: "/worker/" } - route: - cluster: worker-service - prefix_rewrite: "/" - - - match: { prefix: "/db/" } - route: - cluster: couchdb-service - prefix_rewrite: "/" - - # minio is on the default route because this works - # best, minio + AWS SDK doesn't handle path proxy - - match: { prefix: "/" } - route: - cluster: minio-service - - http_filters: - - name: envoy.filters.http.router - - clusters: - - name: app-service - connect_timeout: 0.25s - type: strict_dns - lb_policy: round_robin - load_assignment: - cluster_name: app-service - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: app-service - port_value: 4002 - - - name: minio-service - connect_timeout: 0.25s - type: strict_dns - lb_policy: round_robin - load_assignment: - cluster_name: minio-service - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: minio-service - port_value: 9000 - - - name: worker-service - connect_timeout: 0.25s - type: strict_dns - lb_policy: round_robin - load_assignment: - cluster_name: worker-service - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: worker-service - port_value: 4003 - - - name: couchdb-service - connect_timeout: 0.25s - type: strict_dns - lb_policy: round_robin - load_assignment: - cluster_name: couchdb-service - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: couchdb-service - port_value: 5984 - - - name: watchtower-service - connect_timeout: 0.25s - type: strict_dns - lb_policy: round_robin - load_assignment: - cluster_name: watchtower-service - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: watchtower-service - port_value: 8080 - diff --git a/hosting/hosting.properties b/hosting/hosting.properties deleted file mode 100644 index c8e2f5c60..000000000 --- a/hosting/hosting.properties +++ /dev/null @@ -1,21 +0,0 @@ -# Use the main port in the builder for your self hosting URL, e.g. localhost:10000 -MAIN_PORT=10000 - -# This section contains all secrets pertaining to the system -# These should be updated -JWT_SECRET=testsecret -MINIO_ACCESS_KEY=budibase -MINIO_SECRET_KEY=budibase -COUCH_DB_PASSWORD=budibase -COUCH_DB_USER=budibase -REDIS_PASSWORD=budibase -INTERNAL_API_KEY=budibase - -# This section contains variables that do not need to be altered under normal circumstances -APP_PORT=4002 -WORKER_PORT=4003 -MINIO_PORT=4004 -COUCH_DB_PORT=4005 -REDIS_PORT=6379 -WATCHTOWER_PORT=6161 -BUDIBASE_ENVIRONMENT=PRODUCTION diff --git a/hosting/kubernetes/envoy/Dockerfile b/hosting/kubernetes/envoy/Dockerfile deleted file mode 100644 index 96334fa72..000000000 --- a/hosting/kubernetes/envoy/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM envoyproxy/envoy:v1.16-latest -COPY envoy.yaml /etc/envoy/envoy.yaml -RUN chmod go+r /etc/envoy/envoy.yaml - diff --git a/hosting/kubernetes/envoy/envoy.yaml b/hosting/kubernetes/envoy/envoy.yaml deleted file mode 100644 index bab1f25c0..000000000 --- a/hosting/kubernetes/envoy/envoy.yaml +++ /dev/null @@ -1,146 +0,0 @@ -static_resources: - listeners: - - name: main_listener - address: - socket_address: { address: 0.0.0.0, port_value: 10000 } - filter_chains: - - filters: - - name: envoy.filters.network.http_connection_manager - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager - stat_prefix: ingress - codec_type: auto - route_config: - name: local_route - virtual_hosts: - - name: local_services - domains: ["*"] - routes: - - match: { prefix: "/app/" } - route: - cluster: app-service - prefix_rewrite: "/" - - - match: { prefix: "/builder/" } - route: - cluster: app-service - - - match: { prefix: "/builder" } - route: - cluster: app-service - - - match: { prefix: "/app_" } - route: - cluster: app-service - - # special cases for worker admin (deprecated), global and system API - - match: { prefix: "/api/global/" } - route: - cluster: worker-service - - - match: { prefix: "/api/admin/" } - route: - cluster: worker-service - - - match: { prefix: "/api/system/" } - route: - cluster: worker-service - - - match: { path: "/" } - route: - cluster: app-service - - - match: - safe_regex: - google_re2: {} - regex: "/api/.*/export" - route: - timeout: 0s - cluster: app-service - - - match: { path: "/api/deploy" } - route: - timeout: 60s - cluster: app-service - - # special case for when API requests are made, can just forward, not to minio - - match: { prefix: "/api/" } - route: - cluster: app-service - - - match: { prefix: "/worker/" } - route: - cluster: worker-service - prefix_rewrite: "/" - - - match: { prefix: "/db/" } - route: - cluster: couchdb-service - prefix_rewrite: "/" - - # minio is on the default route because this works - # best, minio + AWS SDK doesn't handle path proxy - - match: { prefix: "/" } - route: - cluster: minio-service - - http_filters: - - name: envoy.filters.http.router - - clusters: - - name: app-service - connect_timeout: 0.25s - type: strict_dns - lb_policy: round_robin - load_assignment: - cluster_name: app-service - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: app-service.budibase.svc.cluster.local - port_value: 4002 - - - name: minio-service - connect_timeout: 0.25s - type: strict_dns - lb_policy: round_robin - load_assignment: - cluster_name: minio-service - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: minio-service.budibase.svc.cluster.local - port_value: 9000 - - - name: worker-service - connect_timeout: 0.25s - type: strict_dns - lb_policy: round_robin - load_assignment: - cluster_name: worker-service - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: worker-service.budibase.svc.cluster.local - port_value: 4001 - - - name: couchdb-service - connect_timeout: 0.25s - type: strict_dns - lb_policy: round_robin - load_assignment: - cluster_name: couchdb-service - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: budibase-prod-svc-couchdb - port_value: 5984 - diff --git a/hosting/kubernetes/nginx/Dockerfile b/hosting/kubernetes/nginx/Dockerfile new file mode 100644 index 000000000..754f9f9be --- /dev/null +++ b/hosting/kubernetes/nginx/Dockerfile @@ -0,0 +1,2 @@ +FROM nginx:latest +COPY nginx.conf /etc/nginx/nginx.conf diff --git a/hosting/kubernetes/nginx/nginx.conf b/hosting/kubernetes/nginx/nginx.conf new file mode 100644 index 000000000..2bf512964 --- /dev/null +++ b/hosting/kubernetes/nginx/nginx.conf @@ -0,0 +1,127 @@ +user nginx; +error_log /var/log/nginx/error.log debug; +pid /var/run/nginx.pid; +worker_processes auto; +worker_rlimit_nofile 33282; + +events { + worker_connections 1024; +} + +http { + limit_req_zone $binary_remote_addr zone=ratelimit:10m rate=10r/s; + include /etc/nginx/mime.types; + default_type application/octet-stream; + charset utf-8; + sendfile on; + tcp_nopush on; + tcp_nodelay on; + server_tokens off; + types_hash_max_size 2048; + + # buffering + client_body_buffer_size 1K; + client_header_buffer_size 1k; + client_max_body_size 1k; + ignore_invalid_headers off; + proxy_buffering off; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + map $http_upgrade $connection_upgrade { + default "upgrade"; + } + + server { + listen 10000 default_server; + server_name _; + + # Security Headers + add_header X-Frame-Options SAMEORIGIN always; + add_header X-Content-Type-Options nosniff always; + add_header X-XSS-Protection "1; mode=block" always; + add_header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval' https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io; style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me; object-src 'none'; base-uri 'self'; connect-src 'self' https://api-iam.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io; font-src 'self' https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me; frame-src 'self'; img-src https: data:; manifest-src 'self'; media-src 'self'; worker-src 'none';" always; + + location /app { + proxy_pass http://app-service:4002; + rewrite ^/app/(.*)$ /$1 break; + } + + location = / { + proxy_pass http://app-service.budibase.svc.cluster.local:4002; + } + + location /builder/ { + proxy_http_version 1.1; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_pass http://app-service.budibase.svc.cluster.local:4002; + } + + location ^/(builder|app_) { + proxy_pass http://app-service.budibase.svc.cluster.local:4002; + } + + location ~ ^/api/(system|admin|global)/ { + proxy_pass http://worker-service.budibase.svc.cluster.local:4003; + } + + location /worker/ { + proxy_pass http://worker-service.budibase.svc.cluster.local:4003; + rewrite ^/worker/(.*)$ /$1 break; + } + + location /api/ { + # calls to the API are rate limited with bursting + limit_req zone=ratelimit burst=10 nodelay; + + # 120s timeout on API requests + proxy_read_timeout 120s; + proxy_connect_timeout 120s; + proxy_send_timeout 120s; + + proxy_http_version 1.1; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + proxy_pass http://app-service.budibase.svc.cluster.local:4002; + } + + location /db/ { + proxy_pass http://budibase-prod-svc-couchdb:5984; + rewrite ^/db/(.*)$ /$1 break; + } + + location / { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + + proxy_connect_timeout 300; + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + proxy_pass http://minio-service.budibase.svc.cluster.local:9000; + } + + client_header_timeout 60; + client_body_timeout 60; + keepalive_timeout 60; + + # gzip + gzip on; + gzip_vary on; + gzip_proxied any; + gzip_comp_level 6; + gzip_types text/plain text/css text/xml application/json application/javascript application/rss+xml application/atom+xml image/svg+xml; + } +} diff --git a/hosting/nginx.dev.conf.hbs b/hosting/nginx.dev.conf.hbs new file mode 100644 index 000000000..51b55cd49 --- /dev/null +++ b/hosting/nginx.dev.conf.hbs @@ -0,0 +1,91 @@ +user nginx; +error_log /var/log/nginx/error.log debug; +pid /var/run/nginx.pid; +worker_processes auto; +worker_rlimit_nofile 33282; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + map $http_upgrade $connection_upgrade { + default "upgrade"; + } + + server { + listen 10000 default_server; + server_name _; + client_max_body_size 1000m; + ignore_invalid_headers off; + proxy_buffering off; + + location /db/ { + proxy_pass http://couchdb-service:5984; + rewrite ^/db/(.*)$ /$1 break; + } + + location ~ ^/api/(system|admin|global)/ { + proxy_pass http://{{ address }}:4002; + } + + location /api/ { + proxy_read_timeout 120s; + proxy_connect_timeout 120s; + proxy_send_timeout 120s; + proxy_pass http://{{ address }}:4001; + } + + location /app_ { + proxy_pass http://{{ address }}:4001; + } + + location /app/ { + proxy_pass http://{{ address }}:4001; + rewrite ^/app/(.*)$ /$1 break; + } + + location /builder { + proxy_pass http://{{ address }}:3000; + rewrite ^/builder(.*)$ /builder/$1 break; + } + + location /builder/ { + proxy_pass http://{{ address }}:3000; + + proxy_http_version 1.1; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location / { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + + proxy_connect_timeout 300; + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + proxy_pass http://minio-service:9000; + } + + client_header_timeout 60; + client_body_timeout 60; + keepalive_timeout 60; + gzip off; + gzip_comp_level 4; + } +} \ No newline at end of file diff --git a/hosting/proxy/Dockerfile b/hosting/proxy/Dockerfile new file mode 100644 index 000000000..2fd64a9d6 --- /dev/null +++ b/hosting/proxy/Dockerfile @@ -0,0 +1,2 @@ +FROM nginx:latest +COPY nginx.conf /etc/nginx/nginx.conf \ No newline at end of file diff --git a/hosting/proxy/nginx.conf b/hosting/proxy/nginx.conf new file mode 100644 index 000000000..7a8a44e2d --- /dev/null +++ b/hosting/proxy/nginx.conf @@ -0,0 +1,135 @@ +user nginx; +error_log /var/log/nginx/error.log debug; +pid /var/run/nginx.pid; +worker_processes auto; +worker_rlimit_nofile 33282; + +events { + worker_connections 1024; +} + +http { + limit_req_zone $binary_remote_addr zone=ratelimit:10m rate=10r/s; + include /etc/nginx/mime.types; + default_type application/octet-stream; + charset utf-8; + sendfile on; + tcp_nopush on; + tcp_nodelay on; + server_tokens off; + types_hash_max_size 2048; + + # buffering + client_body_buffer_size 1K; + client_header_buffer_size 1k; + client_max_body_size 1k; + ignore_invalid_headers off; + proxy_buffering off; + + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + map $http_upgrade $connection_upgrade { + default "upgrade"; + } + + server { + listen 10000 default_server; + server_name _; + client_max_body_size 1000m; + ignore_invalid_headers off; + proxy_buffering off; + + # Security Headers + add_header X-Frame-Options SAMEORIGIN always; + add_header X-Content-Type-Options nosniff always; + add_header X-XSS-Protection "1; mode=block" always; + add_header Content-Security-Policy "default-src 'self'; script-src 'unsafe-inline' 'unsafe-eval' 'self' https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io; style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me; object-src 'none'; base-uri 'self'; connect-src 'self' https://api-iam.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io; font-src 'self' https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me; frame-src 'self'; img-src https: data:; manifest-src 'self'; media-src 'self'; worker-src 'none';" always; + + location /app { + proxy_pass http://app-service:4002; + rewrite ^/app/(.*)$ /$1 break; + } + + location = / { + proxy_pass http://app-service:4002; + } + + location = /v1/update { + proxy_pass http://watchtower-service:8080; + } + + location /builder/ { + proxy_http_version 1.1; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_pass http://app-service:4002; + } + + location ^/(builder|app_) { + proxy_pass http://app-service:4002; + } + + location ~ ^/api/(system|admin|global)/ { + proxy_pass http://worker-service:4003; + } + + location /worker/ { + proxy_pass http://worker-service:4003; + rewrite ^/worker/(.*)$ /$1 break; + } + + location /api/ { + # calls to the API are rate limited with bursting + limit_req zone=ratelimit burst=10 nodelay; + + # 120s timeout on API requests + proxy_read_timeout 120s; + proxy_connect_timeout 120s; + proxy_send_timeout 120s; + + proxy_http_version 1.1; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + proxy_pass http://app-service:4002; + } + + location /db/ { + proxy_pass http://couchdb-service:5984; + rewrite ^/db/(.*)$ /$1 break; + } + + location / { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + + proxy_connect_timeout 300; + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + proxy_pass http://minio-service:9000; + } + + client_header_timeout 60; + client_body_timeout 60; + keepalive_timeout 60; + + # gzip + gzip on; + gzip_vary on; + gzip_proxied any; + gzip_comp_level 6; + gzip_types text/plain text/css text/xml application/json application/javascript application/rss+xml application/atom+xml image/svg+xml; + } +} \ No newline at end of file diff --git a/hosting/scripts/airgapped/airgappedDockerBuild.js b/hosting/scripts/airgapped/airgappedDockerBuild.js index 5be19fdeb..4bd324364 100755 --- a/hosting/scripts/airgapped/airgappedDockerBuild.js +++ b/hosting/scripts/airgapped/airgappedDockerBuild.js @@ -5,7 +5,7 @@ const path = require("path") const IMAGES = { worker: "budibase/worker", apps: "budibase/apps", - proxy: "envoyproxy/envoy:v1.16-latest", + proxy: "budibase/proxy", minio: "minio/minio", couch: "ibmcom/couchdb3", curl: "curlimages/curl", @@ -15,8 +15,7 @@ const IMAGES = { const FILES = { COMPOSE: "docker-compose.yaml", - ENVOY: "envoy.yaml", - PROPERTIES: "hosting.properties" + NGINX: "nginx.conf" } const OUTPUT_DIR = path.join(__dirname, "../", "bb-airgapped") diff --git a/hosting/scripts/linux/release-to-docker-hub.sh b/hosting/scripts/linux/release-to-docker-hub.sh index 642a8682f..599a10f91 100755 --- a/hosting/scripts/linux/release-to-docker-hub.sh +++ b/hosting/scripts/linux/release-to-docker-hub.sh @@ -9,8 +9,10 @@ fi echo "Tagging images with tag: $tag" +docker tag proxy-service budibase/proxy:$tag docker tag app-service budibase/apps:$tag docker tag worker-service budibase/worker:$tag docker push --all-tags budibase/apps docker push --all-tags budibase/worker +docker push --all-tags budibase/proxy diff --git a/lerna.json b/lerna.json index ef5aa714e..f23d6eb65 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "1.0.50-alpha.5", + "version": "1.0.58-alpha.2", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/package.json b/package.json index 5960d15e7..9ab9a4411 100644 --- a/package.json +++ b/package.json @@ -44,9 +44,10 @@ "lint:fix": "yarn run lint:fix:ts && yarn run lint:fix:prettier && yarn run lint:fix:eslint", "test:e2e": "lerna run cy:test", "test:e2e:ci": "lerna run cy:ci", - "build:docker": "lerna run build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -", + "build:docker": "lerna run build:docker && npm run build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -", + "build:docker:proxy": "docker build hosting/proxy -t proxy-service", "build:docker:selfhost": "lerna run build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh latest && cd -", - "build:docker:develop": "node scripts/pinVersions && lerna run build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -", + "build:docker:develop": "node scripts/pinVersions && lerna run build:docker && npm run build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -", "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild", "build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -", "build:docs": "lerna run build:docs", diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index ffb827746..abb142d5f 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "1.0.50-alpha.5", + "version": "1.0.58-alpha.2", "description": "Budibase backend core libraries used in server and worker", "main": "src/index.js", "author": "Budibase", diff --git a/packages/backend-core/src/environment.js b/packages/backend-core/src/environment.js index c26ad1c19..d112ad859 100644 --- a/packages/backend-core/src/environment.js +++ b/packages/backend-core/src/environment.js @@ -11,6 +11,8 @@ module.exports = { COUCH_DB_URL: process.env.COUCH_DB_URL, COUCH_DB_USERNAME: process.env.COUCH_DB_USER, COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD, + GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID, + GOOGLE_CLIENT_SECRET: process.env.GOOGLE_CLIENT_SECRET, SALT_ROUNDS: process.env.SALT_ROUNDS, REDIS_URL: process.env.REDIS_URL, REDIS_PASSWORD: process.env.REDIS_PASSWORD, diff --git a/packages/backend-core/src/middleware/passport/datasource/google.js b/packages/backend-core/src/middleware/passport/datasource/google.js index bfc2e4a61..c7553cee5 100644 --- a/packages/backend-core/src/middleware/passport/datasource/google.js +++ b/packages/backend-core/src/middleware/passport/datasource/google.js @@ -1,22 +1,17 @@ -const { getScopedConfig } = require("../../../db/utils") -const { getGlobalDB } = require("../../../tenancy") const google = require("../google") -const { Configs, Cookies } = require("../../../constants") +const { Cookies } = require("../../../constants") const { clearCookie, getCookie } = require("../../../utils") const { getDB } = require("../../../db") +const environment = require("../../../environment") async function preAuth(passport, ctx, next) { - const db = getGlobalDB() // get the relevant config - const config = await getScopedConfig(db, { - type: Configs.GOOGLE, - workspace: ctx.query.workspace, - }) - const publicConfig = await getScopedConfig(db, { - type: Configs.SETTINGS, - }) - let callbackUrl = `${publicConfig.platformUrl}/api/global/auth/datasource/google/callback` - const strategy = await google.strategyFactory(config, callbackUrl) + const googleConfig = { + clientID: environment.GOOGLE_CLIENT_ID, + clientSecret: environment.GOOGLE_CLIENT_SECRET, + } + let callbackUrl = `${environment.PLATFORM_URL}/api/global/auth/datasource/google/callback` + const strategy = await google.strategyFactory(googleConfig, callbackUrl) if (!ctx.query.appId || !ctx.query.datasourceId) { ctx.throw(400, "appId and datasourceId query params not present.") @@ -30,18 +25,13 @@ async function preAuth(passport, ctx, next) { } async function postAuth(passport, ctx, next) { - const db = getGlobalDB() - - const config = await getScopedConfig(db, { - type: Configs.GOOGLE, - workspace: ctx.query.workspace, - }) - - const publicConfig = await getScopedConfig(db, { - type: Configs.SETTINGS, - }) + // get the relevant config + const config = { + clientID: environment.GOOGLE_CLIENT_ID, + clientSecret: environment.GOOGLE_CLIENT_SECRET, + } - let callbackUrl = `${publicConfig.platformUrl}/api/global/auth/datasource/google/callback` + let callbackUrl = `${environment.PLATFORM_URL}/api/global/auth/datasource/google/callback` const strategy = await google.strategyFactory( config, callbackUrl, diff --git a/packages/bbui/package.json b/packages/bbui/package.json index f9be4f22d..99efd30b8 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "1.0.50-alpha.5", + "version": "1.0.58-alpha.2", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", diff --git a/packages/builder/cypress/setup.js b/packages/builder/cypress/setup.js index 0f3d333cb..ca5a65c7f 100644 --- a/packages/builder/cypress/setup.js +++ b/packages/builder/cypress/setup.js @@ -15,7 +15,7 @@ process.env.COUCH_URL = `leveldb://${tmpdir}/.data/` process.env.SELF_HOSTED = 1 process.env.WORKER_URL = "http://localhost:10002/" process.env.APPS_URL = `http://localhost:${MAIN_PORT}/` -process.env.MINIO_URL = `http://localhost:${MAIN_PORT}/` +process.env.MINIO_URL = `http://localhost:4004` process.env.MINIO_ACCESS_KEY = "budibase" process.env.MINIO_SECRET_KEY = "budibase" process.env.COUCH_DB_USER = "budibase" diff --git a/packages/builder/package.json b/packages/builder/package.json index 91e568da6..4c85293ef 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "1.0.50-alpha.5", + "version": "1.0.58-alpha.2", "license": "GPL-3.0", "private": true, "scripts": { @@ -64,10 +64,10 @@ } }, "dependencies": { - "@budibase/bbui": "^1.0.50-alpha.5", - "@budibase/client": "^1.0.50-alpha.5", - "@budibase/frontend-core": "^1.0.50-alpha.5", - "@budibase/string-templates": "^1.0.50-alpha.5", + "@budibase/bbui": "^1.0.58-alpha.2", + "@budibase/client": "^1.0.58-alpha.2", + "@budibase/frontend-core": "^1.0.58-alpha.2", + "@budibase/string-templates": "^1.0.58-alpha.2", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleButton.svelte b/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleButton.svelte index 091e33283..642aa548f 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleButton.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleButton.svelte @@ -17,7 +17,7 @@ ds = await preAuthStep() } window.open( - `/api/global/auth/${tenantId}/datasource/google?datasourceId=${datasource._id}&appId=${$store.appId}`, + `/api/global/auth/${tenantId}/datasource/google?datasourceId=${ds._id}&appId=${$store.appId}`, "_blank" ) }} diff --git a/packages/cli/.gitignore b/packages/cli/.gitignore index c47f898f2..618aa6638 100644 --- a/packages/cli/.gitignore +++ b/packages/cli/.gitignore @@ -1,6 +1,5 @@ node_modules/ docker-compose.yaml -envoy.yaml -hosting.properties +nginx.conf build/ docker-error.log diff --git a/packages/cli/package.json b/packages/cli/package.json index 8a1ce573f..0881c783d 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "1.0.50-alpha.5", + "version": "1.0.58-alpha.2", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/cli/src/hosting/index.js b/packages/cli/src/hosting/index.js index 05d221435..ecf3b710b 100644 --- a/packages/cli/src/hosting/index.js +++ b/packages/cli/src/hosting/index.js @@ -19,7 +19,6 @@ const BUDIBASE_SERVICES = ["app-service", "worker-service"] const ERROR_FILE = "docker-error.log" const FILE_URLS = [ "https://raw.githubusercontent.com/Budibase/budibase/master/hosting/docker-compose.yaml", - "https://raw.githubusercontent.com/Budibase/budibase/master/hosting/envoy.yaml", ] const DO_USER_DATA_URL = "http://169.254.169.254/metadata/v1/user-data" @@ -141,11 +140,7 @@ async function stop() { async function update() { await checkDockerConfigured() checkInitComplete() - if ( - await confirmation( - "Do you wish to update you docker-compose.yaml and envoy.yaml?" - ) - ) { + if (await confirmation("Do you wish to update you docker-compose.yaml?")) { await downloadFiles() } await handleError(async () => { diff --git a/packages/client/manifest.json b/packages/client/manifest.json index 29cbc0536..06dbaad66 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -2946,7 +2946,7 @@ "settings": [ { "type": "number", - "label": "Row Count", + "label": "Scroll Limit", "key": "rowCount", "defaultValue": 8 }, diff --git a/packages/client/package.json b/packages/client/package.json index ac01b8367..dcdf6f6c6 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "1.0.50-alpha.5", + "version": "1.0.58-alpha.2", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^1.0.50-alpha.5", - "@budibase/frontend-core": "^1.0.50-alpha.5", - "@budibase/string-templates": "^1.0.50-alpha.5", + "@budibase/bbui": "^1.0.58-alpha.2", + "@budibase/frontend-core": "^1.0.58-alpha.2", + "@budibase/string-templates": "^1.0.58-alpha.2", "@spectrum-css/button": "^3.0.3", "@spectrum-css/card": "^3.0.3", "@spectrum-css/divider": "^1.0.3", diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index 4f2770c71..376cc470b 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,12 +1,12 @@ { "name": "@budibase/frontend-core", - "version": "1.0.50-alpha.5", + "version": "1.0.58-alpha.2", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "^1.0.50-alpha.4", + "@budibase/bbui": "^1.0.58-alpha.2", "lodash": "^4.17.21", "svelte": "^3.46.2" } diff --git a/packages/server/package.json b/packages/server/package.json index 171f1b74b..238ca5fcf 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "1.0.50-alpha.5", + "version": "1.0.58-alpha.2", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -70,9 +70,9 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "^10.0.3", - "@budibase/backend-core": "^1.0.50-alpha.5", - "@budibase/client": "^1.0.50-alpha.5", - "@budibase/string-templates": "^1.0.50-alpha.5", + "@budibase/backend-core": "^1.0.58-alpha.2", + "@budibase/client": "^1.0.58-alpha.2", + "@budibase/string-templates": "^1.0.58-alpha.2", "@bull-board/api": "^3.7.0", "@bull-board/koa": "^3.7.0", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/server/scripts/dev/manage.js b/packages/server/scripts/dev/manage.js index f3938ad8e..77a09a482 100644 --- a/packages/server/scripts/dev/manage.js +++ b/packages/server/scripts/dev/manage.js @@ -22,21 +22,21 @@ const Commands = { } async function init() { - // generate envoy file, always do this incase it has changed + // generate nginx file, always do this incase it has changed const hostingPath = path.join(process.cwd(), "..", "..", "hosting") - const envoyHbsPath = path.join(hostingPath, "envoy.dev.yaml.hbs") - const envoyOutputPath = path.join(hostingPath, ".generated-envoy.dev.yaml") - const contents = fs.readFileSync(envoyHbsPath, "utf8") + const nginxHbsPath = path.join(hostingPath, "nginx.dev.conf.hbs") + const nginxOutputPath = path.join(hostingPath, ".generated-nginx.dev.conf") + const contents = fs.readFileSync(nginxHbsPath, "utf8") const config = { address: isLinux() ? "172.17.0.1" : "host.docker.internal", } - fs.writeFileSync(envoyOutputPath, processStringSync(contents, config)) + fs.writeFileSync(nginxOutputPath, processStringSync(contents, config)) const envFilePath = path.join(process.cwd(), ".env") if (!fs.existsSync(envFilePath)) { const envFileJson = { PORT: 4001, - MINIO_URL: "http://localhost:10000/", + MINIO_URL: "http://localhost:4004", COUCH_DB_URL: "http://budibase:budibase@localhost:10000/db/", REDIS_URL: "localhost:6379", WORKER_URL: "http://localhost:4002", diff --git a/packages/server/scripts/likeCypress.ts b/packages/server/scripts/likeCypress.ts index 834cf3279..2ca4933ed 100644 --- a/packages/server/scripts/likeCypress.ts +++ b/packages/server/scripts/likeCypress.ts @@ -17,7 +17,7 @@ process.env.JWT_SECRET = "budibase" process.env.COUCH_URL = `leveldb://${tmpdir}/.data/` process.env.SELF_HOSTED = "1" process.env.WORKER_URL = `http://localhost:${WORKER_PORT}/` -process.env.MINIO_URL = `http://localhost:${MAIN_PORT}/` +process.env.MINIO_URL = `http://localhost:4004` process.env.MINIO_ACCESS_KEY = "budibase" process.env.MINIO_SECRET_KEY = "budibase" process.env.COUCH_DB_USER = "budibase" diff --git a/packages/server/src/api/controllers/integration.js b/packages/server/src/api/controllers/integration.js index f1aafe7e4..75fd9ce80 100644 --- a/packages/server/src/api/controllers/integration.js +++ b/packages/server/src/api/controllers/integration.js @@ -1,7 +1,6 @@ const { definitions } = require("../../integrations") exports.fetch = async function (ctx) { - // TODO: fetch these from a github repo etc ctx.status = 200 ctx.body = definitions } diff --git a/packages/server/src/api/controllers/query/validation.js b/packages/server/src/api/controllers/query/validation.js index 495843384..515c7b591 100644 --- a/packages/server/src/api/controllers/query/validation.js +++ b/packages/server/src/api/controllers/query/validation.js @@ -31,8 +31,14 @@ exports.generateQueryValidation = () => { exports.generateQueryPreviewValidation = () => { // prettier-ignore return joiValidator.body(Joi.object({ + _id: Joi.string().optional(), + _rev: Joi.string().optional(), + readable: Joi.boolean().optional(), fields: Joi.object().required(), queryVerb: Joi.string().allow().required(), + name: Joi.string().required(), + flags: Joi.object().optional(), + schema: Joi.object().optional(), extra: Joi.object().optional(), datasourceId: Joi.string().required(), transformer: Joi.string().optional(), diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 6aa51fb36..c1181dd6d 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -541,7 +541,7 @@ module External { if (!linkTable || !linkPrimary) { return } - const rows = related[key].rows || [] + const rows = related[key]?.rows || [] const found = rows.find( (row: { [key: string]: any }) => row[linkPrimary] === relationship.id || diff --git a/packages/server/src/api/routes/tests/query.spec.js b/packages/server/src/api/routes/tests/query.spec.js index dac576836..48e3181d1 100644 --- a/packages/server/src/api/routes/tests/query.spec.js +++ b/packages/server/src/api/routes/tests/query.spec.js @@ -169,6 +169,7 @@ describe("/queries", () => { parameters: {}, fields: {}, queryVerb: "read", + name: datasource.name, }) .set(config.defaultHeaders()) .expect("Content-Type", /json/) @@ -261,9 +262,13 @@ describe("/queries", () => { }) it("check that it automatically retries on fail with cached dynamics", async () => { - const { datasource, query: base } = await config.dynamicVariableDatasource() + const { datasource, query: base } = + await config.dynamicVariableDatasource() // preview once to cache - await preview(datasource, { path: "www.google.com", queryString: "test={{ variable3 }}" }) + await preview(datasource, { + path: "www.google.com", + queryString: "test={{ variable3 }}", + }) // check its in cache const contents = await checkCacheForDynamicVariable(base._id, "variable3") expect(contents.rows.length).toEqual(1) @@ -276,9 +281,13 @@ describe("/queries", () => { }) it("deletes variables when linked query is deleted", async () => { - const { datasource, query: base } = await config.dynamicVariableDatasource() + const { datasource, query: base } = + await config.dynamicVariableDatasource() // preview once to cache - await preview(datasource, { path: "www.google.com", queryString: "test={{ variable3 }}" }) + await preview(datasource, { + path: "www.google.com", + queryString: "test={{ variable3 }}", + }) // check its in cache let contents = await checkCacheForDynamicVariable(base._id, "variable3") expect(contents.rows.length).toEqual(1) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index a4220565c..ce0662410 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -210,49 +210,37 @@ class InternalBuilder { const { toTable, throughTable } = JSON.parse(key) if (!throughTable) { // @ts-ignore - query = query.join( - toTable, - function () { - for (let relationship of relationships) { - const from = relationship.from, - to = relationship.to - // @ts-ignore - this.orOn(`${fromTable}.${from}`, "=", `${toTable}.${to}`) - } - }, - "left" - ) + query = query.leftJoin(toTable, function () { + for (let relationship of relationships) { + const from = relationship.from, + to = relationship.to + // @ts-ignore + this.orOn(`${fromTable}.${from}`, "=", `${toTable}.${to}`) + } + }) } else { query = query // @ts-ignore - .join( - throughTable, - function () { - for (let relationship of relationships) { - const fromPrimary = relationship.fromPrimary - const from = relationship.from - // @ts-ignore - this.orOn( - `${fromTable}.${fromPrimary}`, - "=", - `${throughTable}.${from}` - ) - } - }, - "left" - ) - .join( - toTable, - function () { - for (let relationship of relationships) { - const toPrimary = relationship.toPrimary - const to = relationship.to - // @ts-ignore - this.orOn(`${toTable}.${toPrimary}`, `${throughTable}.${to}`) - } - }, - "left" - ) + .leftJoin(throughTable, function () { + for (let relationship of relationships) { + const fromPrimary = relationship.fromPrimary + const from = relationship.from + // @ts-ignore + this.orOn( + `${fromTable}.${fromPrimary}`, + "=", + `${throughTable}.${from}` + ) + } + }) + .leftJoin(toTable, function () { + for (let relationship of relationships) { + const toPrimary = relationship.toPrimary + const to = relationship.to + // @ts-ignore + this.orOn(`${toTable}.${toPrimary}`, `${throughTable}.${to}`) + } + }) } } return query.limit(BASE_LIMIT) diff --git a/packages/server/src/integrations/dynamodb.ts b/packages/server/src/integrations/dynamodb.ts index bfcac3184..841dd6ff0 100644 --- a/packages/server/src/integrations/dynamodb.ts +++ b/packages/server/src/integrations/dynamodb.ts @@ -169,13 +169,11 @@ module DynamoModule { IndexName: query.index ? query.index : undefined, ...query.json, } - if (query.index) { - const response = await this.client.query(params).promise() - if (response.Items) { - return response.Items - } - return response + const response = await this.client.query(params).promise() + if (response.Items) { + return response.Items } + return response } async scan(query: { table: string; json: object; index: null | string }) { diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index 4679e658b..00b00c25f 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -11,6 +11,7 @@ const arangodb = require("./arangodb") const rest = require("./rest") const googlesheets = require("./googlesheets") const { SourceNames } = require("../definitions/datasource") +const environment = require("../environment") const DEFINITIONS = { [SourceNames.POSTGRES]: postgres.schema, @@ -24,7 +25,6 @@ const DEFINITIONS = { [SourceNames.MYSQL]: mysql.schema, [SourceNames.ARANGODB]: arangodb.schema, [SourceNames.REST]: rest.schema, - [SourceNames.GOOGLE_SHEETS]: googlesheets.schema, } const INTEGRATIONS = { @@ -39,7 +39,6 @@ const INTEGRATIONS = { [SourceNames.MYSQL]: mysql.integration, [SourceNames.ARANGODB]: arangodb.integration, [SourceNames.REST]: rest.integration, - [SourceNames.GOOGLE_SHEETS]: googlesheets.integration, } // optionally add oracle integration if the oracle binary can be installed @@ -49,6 +48,11 @@ if (!(process.arch === "arm64" && process.platform === "darwin")) { INTEGRATIONS[SourceNames.ORACLE] = oracle.integration } +if (environment.SELF_HOSTED) { + DEFINITIONS[SourceNames.GOOGLE_SHEETS] = googlesheets.schema + INTEGRATIONS[SourceNames.GOOGLE_SHEETS] = googlesheets.integration +} + module.exports = { definitions: DEFINITIONS, integrations: INTEGRATIONS, diff --git a/packages/server/src/tests/utilities/TestConfiguration.js b/packages/server/src/tests/utilities/TestConfiguration.js index 6c2b7d4f9..1b3a91cea 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.js +++ b/packages/server/src/tests/utilities/TestConfiguration.js @@ -393,6 +393,7 @@ class TestConfiguration { parameters: {}, fields, queryVerb: "read", + name: datasource.name, }) .set(config.defaultHeaders()) .expect("Content-Type", /json/) diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index ec1052b0b..2856286d0 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -983,10 +983,10 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@^1.0.50-alpha.1": - version "1.0.50" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.50.tgz#f31bb365b938c6f889324be7033673d75ea5e127" - integrity sha512-C71kCWdFSrIIkrEuqgRKu2IKyPftK4Rq0HQLOHdnPfmYOGg8iJkeVAXeTQdowaoBXrfzUt/YZmA5ksGccxm3WQ== +"@budibase/backend-core@^1.0.50-alpha.6": + version "1.0.56" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.56.tgz#898ad4df1d923527cb340ae76ec548d1e5234755" + integrity sha512-Cos2TgI6grgSTiLPGlAZb53BEWPrzU3e9UXhhTtwiROraYxkVQob6kOQMK7Cvnm3IC+viXjDRijysvactlRwFA== dependencies: "@techpass/passport-openidconnect" "^0.3.0" aws-sdk "^2.901.0" @@ -1056,10 +1056,10 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/bbui@^1.0.50": - version "1.0.50" - resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.0.50.tgz#f49f0e2255647ca36306ab44ed565d1962f1b27a" - integrity sha512-9k7SBKq5tIO0/w8uWjbFrLIeMhdgCMeEXNGS1gXON4pt3ydSblKuWdRSPwSs8BZNfUMid+UxPkkdtLgZIBKcPA== +"@budibase/bbui@^1.0.56": + version "1.0.56" + resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.0.56.tgz#3c8cd78c97a21a34fc4b9a45483c7eca739a1dd8" + integrity sha512-HNcQFSFyvtQBtDvqLhQwUCGVJsvNmRt5GZsgIZr73oKaowiC6cwBQm/uPg3I9GxqrTz6etPy0E5ELm/cZKptxA== dependencies: "@adobe/spectrum-css-workflow-icons" "^1.2.1" "@spectrum-css/actionbutton" "^1.0.1" @@ -1107,14 +1107,14 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/client@^1.0.50-alpha.1": - version "1.0.50" - resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.50.tgz#513295389572c48269716c816204443aa215274e" - integrity sha512-pKLXPJLNQpKeoNixoQp1WsQWUTzucfNPuZNjCZyyCCavToj9kZArvgxAYRcJGgWAulqLDMAr+bC1L5UNJ25Tyw== +"@budibase/client@^1.0.50-alpha.6": + version "1.0.56" + resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.56.tgz#58376ee2d6f64d1a37e6cc8898006a74f5f810b2" + integrity sha512-a7D+AKzutYk3N5OlKndv4BYtt0vnkDysjNLRtkZnKj3ERwQzI3bERrs+Wp2o5KI3cbZFWYR8x179YYxjBTcnYg== dependencies: - "@budibase/bbui" "^1.0.50" + "@budibase/bbui" "^1.0.56" "@budibase/standard-components" "^0.9.139" - "@budibase/string-templates" "^1.0.50" + "@budibase/string-templates" "^1.0.56" regexparam "^1.3.0" rollup-plugin-polyfill-node "^0.8.0" shortid "^2.2.15" @@ -1163,10 +1163,10 @@ svelte-apexcharts "^1.0.2" svelte-flatpickr "^3.1.0" -"@budibase/string-templates@^1.0.50", "@budibase/string-templates@^1.0.50-alpha.1": - version "1.0.50" - resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.50.tgz#53386f3c09891ef21bd47870d25cf7e12f6fac86" - integrity sha512-jmqmikU3Xt0I0hY1S1QI8x5770Z+Tu6zGxb9iQsk3PUCkKCpHz70Rl3Q7x5nYsDfOlhT4YHaHKgYYQYJ6hRafw== +"@budibase/string-templates@^1.0.50-alpha.6", "@budibase/string-templates@^1.0.56": + version "1.0.56" + resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.56.tgz#13869566e344ec175904e7b535dba31f8ddd723d" + integrity sha512-lynmirU/3v+RPmuLGUG3SQUZ8EhjkaGsGTCtzzJq+59bR72zkpoxk+p2XbtJn5AWZvd4VN7d57w3+wGRMPTzoQ== dependencies: "@budibase/handlebars-helpers" "^0.11.7" dayjs "^1.10.4" diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 4da4029b8..9dc2c6bc0 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "1.0.50-alpha.5", + "version": "1.0.58-alpha.2", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index 4c0f682f2..bfd178b4f 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "1.0.50-alpha.5", + "version": "1.0.58-alpha.2", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -34,8 +34,8 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^1.0.50-alpha.5", - "@budibase/string-templates": "^1.0.50-alpha.5", + "@budibase/backend-core": "^1.0.58-alpha.2", + "@budibase/string-templates": "^1.0.58-alpha.2", "@koa/router": "^8.0.0", "@sentry/node": "^6.0.0", "@techpass/passport-openidconnect": "^0.3.0", diff --git a/packages/worker/scripts/dev/manage.js b/packages/worker/scripts/dev/manage.js index 179167883..34a2b7da0 100644 --- a/packages/worker/scripts/dev/manage.js +++ b/packages/worker/scripts/dev/manage.js @@ -15,7 +15,7 @@ async function init() { MINIO_SECRET_KEY: "budibase", REDIS_URL: "localhost:6379", REDIS_PASSWORD: "budibase", - MINIO_URL: "http://localhost:10000/", + MINIO_URL: "http://localhost:4004", COUCH_DB_URL: "http://budibase:budibase@localhost:10000/db/", COUCH_DB_USERNAME: "budibase", COUCH_DB_PASSWORD: "budibase", diff --git a/packages/worker/src/api/routes/global/auth.js b/packages/worker/src/api/routes/global/auth.js index 373bf5736..cc00b4f82 100644 --- a/packages/worker/src/api/routes/global/auth.js +++ b/packages/worker/src/api/routes/global/auth.js @@ -80,6 +80,11 @@ router updateTenant, authController.googleAuth ) + .get( + "/api/global/auth/:tenantId/datasource/:provider/callback", + updateTenant, + authController.datasourceAuth + ) .get( "/api/global/auth/:tenantId/oidc/configs/:configId", updateTenant,