nginx dev setup

.github/CONTRIBUTING.md

@@ -137,7 +137,7 @@ If you wish to delete all the apps created in development and reset the environm
 
 ### Backend
 
-For the backend we run [Redis](https://redis.io/), [CouchDB](https://couchdb.apache.org/), [MinIO](https://min.io/) and [Envoy](https://www.envoyproxy.io/) in Docker compose. This means that to develop Budibase you will need Docker and Docker compose installed. The backend services are then ran separately as Node services with nodemon so that they can be debugged outside of Docker.
+For the backend we run [Redis](https://redis.io/), [CouchDB](https://couchdb.apache.org/), [MinIO](https://min.io/) and [NGINX](https://www.nginx.com/) in Docker compose. This means that to develop Budibase you will need Docker and Docker compose installed. The backend services are then ran separately as Node services with nodemon so that they can be debugged outside of Docker.
 
 ### Data Storage
 

.gitignore

@@ -64,7 +64,7 @@ typings/
 # dotenv environment variables file
 .env
 !hosting/.env
-hosting/.generated-envoy.dev.yaml
+hosting/.generated-nginx.dev.conf
 
 # parcel-bundler cache (https://parceljs.org/)
 .cache

hosting/digitalocean/files/var/lib/cloud/scripts/per-instance/001_onboot

@@ -3,7 +3,7 @@
 # go into the app dir
 cd /root
 
-# fetch envoy and docker-compose files
+# fetch nginx and docker-compose files
 wget https://raw.githubusercontent.com/Budibase/budibase/master/hosting/docker-compose.yaml
 wget https://raw.githubusercontent.com/Budibase/budibase/master/hosting/envoy.yaml
 wget https://raw.githubusercontent.com/Budibase/budibase/master/hosting/hosting.properties

hosting/docker-compose.dev.yaml

@@ -22,16 +22,18 @@ services:
       retries: 3
 
   proxy-service:
-    container_name: budi-envoy-dev
+    container_name: budi-nginx-dev
     restart: always
-    image: envoyproxy/envoy:v1.16-latest
+    image: nginx:latest
     volumes:
-      - ./.generated-envoy.dev.yaml:/etc/envoy/envoy.yaml
+      - ./.generated-nginx.dev.conf:/etc/nginx/nginx.conf:ro
     ports:
       - "${MAIN_PORT}:10000"
     depends_on:
       - minio-service
       - couchdb-service
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
 
   couchdb-service:
     container_name: budi-couchdb-dev

hosting/kubernetes/nginx/Dockerfile

@@ -0,0 +1,3 @@
+# FROM envoyproxy/envoy:v1.16-latest
+# COPY envoy.yaml /etc/envoy/envoy.yaml
+# RUN chmod go+r /etc/envoy/envoy.yaml

hosting/kubernetes/nginx/nginx.conf

@@ -0,0 +1,146 @@
+static_resources:
+  listeners:
+  - name: main_listener
+    address:
+      socket_address: { address: 0.0.0.0, port_value: 10000 }
+    filter_chains:
+    - filters:
+      - name: envoy.filters.network.http_connection_manager
+        typed_config:
+          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+          stat_prefix: ingress
+          codec_type: auto
+          route_config:
+            name: local_route
+            virtual_hosts:
+            - name: local_services
+              domains: ["*"]
+              routes:
+              - match: { prefix: "/app/" }
+                route:
+                  cluster: app-service
+                  prefix_rewrite: "/"
+
+              - match: { prefix: "/builder/" }
+                route:
+                  cluster: app-service
+
+              - match: { prefix: "/builder" }
+                route:
+                  cluster: app-service
+
+              - match: { prefix: "/app_" }
+                route:
+                  cluster: app-service
+
+              # special cases for worker admin (deprecated), global and system API
+              - match: { prefix: "/api/global/" }
+                route:
+                  cluster: worker-service
+
+              - match: { prefix: "/api/admin/" }
+                route:
+                  cluster: worker-service
+
+              - match: { prefix: "/api/system/" }
+                route:
+                  cluster: worker-service
+
+              - match: { path: "/" }
+                route:
+                  cluster: app-service
+
+              - match: 
+                  safe_regex: 
+                    google_re2: {}
+                    regex: "/api/.*/export"
+                route:
+                  timeout: 0s
+                  cluster: app-service
+
+              - match: { path: "/api/deploy" }
+                route:
+                  timeout: 60s
+                  cluster: app-service
+
+              # special case for when API requests are made, can just forward, not to minio
+              - match: { prefix: "/api/" }
+                route:
+                  cluster: app-service
+
+              - match: { prefix: "/worker/" }
+                route:
+                  cluster: worker-service
+                  prefix_rewrite: "/"
+
+              - match: { prefix: "/db/" }
+                route:
+                  cluster: couchdb-service
+                  prefix_rewrite: "/"
+
+              # minio is on the default route because this works
+              # best, minio + AWS SDK doesn't handle path proxy
+              - match: { prefix: "/" }
+                route:
+                  cluster: minio-service
+
+          http_filters:
+          - name: envoy.filters.http.router
+
+  clusters:
+  - name: app-service
+    connect_timeout: 0.25s
+    type: strict_dns
+    lb_policy: round_robin
+    load_assignment:
+      cluster_name: app-service
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: app-service.budibase.svc.cluster.local
+                port_value: 4002
+
+  - name: minio-service
+    connect_timeout: 0.25s
+    type: strict_dns
+    lb_policy: round_robin
+    load_assignment:
+      cluster_name: minio-service
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: minio-service.budibase.svc.cluster.local
+                port_value: 9000
+
+  - name: worker-service
+    connect_timeout: 0.25s
+    type: strict_dns
+    lb_policy: round_robin
+    load_assignment:
+      cluster_name: worker-service
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: worker-service.budibase.svc.cluster.local
+                port_value: 4001
+
+  - name: couchdb-service
+    connect_timeout: 0.25s
+    type: strict_dns
+    lb_policy: round_robin
+    load_assignment:
+      cluster_name: couchdb-service
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: budibase-prod-svc-couchdb 
+                port_value: 5984
+

hosting/nginx.dev.conf.hbs

@@ -0,0 +1,91 @@
+user                    nginx;
+error_log               /var/log/nginx/error.log warn;
+pid                     /var/run/nginx.pid;
+worker_processes        auto;
+worker_rlimit_nofile    33282;
+
+events {
+  worker_connections  4096;
+}
+
+http {
+  include       /etc/nginx/mime.types;
+  default_type  application/octet-stream;
+
+  log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                    '$status $body_bytes_sent "$http_referer" '
+                    '"$http_user_agent" "$http_x_forwarded_for"';
+
+  map $http_upgrade $connection_upgrade {
+    default     "upgrade";
+  }
+
+  server { 
+    listen       0.0.0.0:10000 default_server;
+    server_name  _;
+    resolver 127.0.0.11 valid=10s;
+
+    # security headers
+    add_header X-Frame-Options SAMEORIGIN always;
+
+    # proxy headers
+    proxy_set_header   Host               $host;
+    proxy_set_header   X-Real-IP          $remote_addr;
+    proxy_set_header   X-Forwarded-Proto  $scheme;
+    proxy_set_header   X-Forwarded-For    $proxy_add_x_forwarded_for;
+
+
+    # budibase services
+    set $app_service http://{{ address }}:4001;
+    set $worker_service http://{{ address }}:4002;
+    set $builder http://{{ address }}:3000;
+    set $minio   http://minio-service:9000;
+    set $couchdb   http://couchdb-service:5984;
+
+    location = / {
+      return 301 /builder/;
+    }
+
+    location /db/ {
+      proxy_pass      $couchdb;
+      rewrite ^/db/(.*)$ /$1 break;
+    }
+
+    location ~ ^/(global|admin|system)/ {
+      proxy_pass      $worker_service;
+    }
+
+    location /api/ {
+      proxy_read_timeout 120s;
+      proxy_connect_timeout 120s;
+      proxy_send_timeout 120s; 
+      proxy_pass      $app_service;
+    }
+
+    location ^/(app_|builder) {
+      proxy_pass      $app_service;
+    }
+
+    location /app/ {
+      proxy_pass      $app_service;
+      rewrite ^/app/(.*)$ /$1 break;
+    }
+
+    {{!-- location = / {
+      proxy_pass      $builder;
+    } --}}
+
+    location /builder {
+      proxy_pass      $builder;
+      rewrite ^/builder(.*)$ /builder/$1 break;
+    }
+
+    location /builder/ {
+      proxy_pass      $builder;
+    }
+
+    location / {
+      proxy_pass      $minio;
+    }
+  }
+}

hosting/proxy/Dockerfile

@@ -0,0 +1,2 @@
+FROM nginx:latest
+COPY nginx.conf /etc/nginx/nginx.conf

hosting/proxy/nginx.conf

@@ -0,0 +1,87 @@
+user nginx;
+worker_processes  5;
+error_log  logs/error.log;
+pid        logs/nginx.pid;
+worker_rlimit_nofile 8192;
+
+events {
+  worker_connections  4096;
+}
+
+http {
+  include       /etc/nginx/mime.types;
+  include       /etc/nginx/proxy.conf;
+  # include    /etc/nginx/fastcgi.conf;
+  # index    index.html index.htm index.php;
+
+  default_type application/octet-stream;
+  log_format   main '$remote_addr - $remote_user [$time_local]  $status '
+    '"$request" $body_bytes_sent "$http_referer" '
+    '"$http_user_agent" "$http_x_forwarded_for"';
+  access_log   logs/access.log  main;
+
+  server { 
+    listen       10000 default_server;
+    listen [::]:10000 default_server;
+    server_name  _;
+    access_log   logs/budiproxy.access.log  main;
+
+    # security headers
+    add_header X-Frame-Options SAMEORIGIN always;
+
+    upstream app-service {
+      server app-service:4002
+    }
+
+    upstream worker-service {
+      server worker-service:4003
+    }
+
+    location = /v1/update {
+      proxy_pass      http://watchtower-service:8080;
+    }
+
+    location /app/ {
+      proxy_pass      http://app-service;
+    }
+
+    # location ^/(app_|builder) {
+    #   proxy_pass      http://app-service:4002;
+    # }
+
+    location /app_ {
+      proxy_pass      http://app-service;
+    }
+
+    location /builder {
+      proxy_pass      http://app-service;
+    }
+
+    location ~ ^/(global|admin|system)/ {
+      proxy_pass      http://worker-service;
+    }
+
+    location = / {
+      proxy_pass      http://app-service;
+    }
+
+    location /api/ {
+      proxy_read_timeout 120s;
+      proxy_connect_timeout 120s;
+      proxy_send_timeout 120s; 
+      proxy_pass      http://app-service;
+    }
+
+    location /worker/ {
+      proxy_pass      http://app-service;
+    }
+
+    location /db/ {
+      proxy_pass      http://couchdb-service:5984;
+    }
+
+    location / {
+      proxy_pass      http://minio-service:9000;
+    }
+  }
+}

packages/server/scripts/dev/manage.js

@@ -24,8 +24,8 @@ const Commands = {
 async function init() {
   // generate envoy file, always do this incase it has changed
   const hostingPath = path.join(process.cwd(), "..", "..", "hosting")
-  const envoyHbsPath = path.join(hostingPath, "envoy.dev.yaml.hbs")
+  const envoyHbsPath = path.join(hostingPath, "nginx.dev.conf.hbs")
-  const envoyOutputPath = path.join(hostingPath, ".generated-envoy.dev.yaml")
+  const envoyOutputPath = path.join(hostingPath, ".generated-nginx.dev.conf")
   const contents = fs.readFileSync(envoyHbsPath, "utf8")
   const config = {
     address: isLinux() ? "172.17.0.1" : "host.docker.internal",