diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index d1e373003..2a57d6f38 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -137,7 +137,7 @@ If you wish to delete all the apps created in development and reset the environm ### Backend -For the backend we run [Redis](https://redis.io/), [CouchDB](https://couchdb.apache.org/), [MinIO](https://min.io/) and [Envoy](https://www.envoyproxy.io/) in Docker compose. This means that to develop Budibase you will need Docker and Docker compose installed. The backend services are then ran separately as Node services with nodemon so that they can be debugged outside of Docker. +For the backend we run [Redis](https://redis.io/), [CouchDB](https://couchdb.apache.org/), [MinIO](https://min.io/) and [NGINX](https://www.nginx.com/) in Docker compose. This means that to develop Budibase you will need Docker and Docker compose installed. The backend services are then ran separately as Node services with nodemon so that they can be debugged outside of Docker. ### Data Storage diff --git a/.gitignore b/.gitignore index 086813c97..4d963e1ab 100644 --- a/.gitignore +++ b/.gitignore @@ -64,7 +64,7 @@ typings/ # dotenv environment variables file .env !hosting/.env -hosting/.generated-envoy.dev.yaml +hosting/.generated-nginx.dev.conf # parcel-bundler cache (https://parceljs.org/) .cache diff --git a/hosting/digitalocean/files/var/lib/cloud/scripts/per-instance/001_onboot b/hosting/digitalocean/files/var/lib/cloud/scripts/per-instance/001_onboot index e5a883ac8..938c5e595 100755 --- a/hosting/digitalocean/files/var/lib/cloud/scripts/per-instance/001_onboot +++ b/hosting/digitalocean/files/var/lib/cloud/scripts/per-instance/001_onboot @@ -3,7 +3,7 @@ # go into the app dir cd /root -# fetch envoy and docker-compose files +# fetch nginx and docker-compose files wget https://raw.githubusercontent.com/Budibase/budibase/master/hosting/docker-compose.yaml wget https://raw.githubusercontent.com/Budibase/budibase/master/hosting/envoy.yaml wget https://raw.githubusercontent.com/Budibase/budibase/master/hosting/hosting.properties diff --git a/hosting/docker-compose.dev.yaml b/hosting/docker-compose.dev.yaml index eaced64e0..9915a7eb1 100644 --- a/hosting/docker-compose.dev.yaml +++ b/hosting/docker-compose.dev.yaml @@ -22,16 +22,18 @@ services: retries: 3 proxy-service: - container_name: budi-envoy-dev + container_name: budi-nginx-dev restart: always - image: envoyproxy/envoy:v1.16-latest + image: nginx:latest volumes: - - ./.generated-envoy.dev.yaml:/etc/envoy/envoy.yaml + - ./.generated-nginx.dev.conf:/etc/nginx/nginx.conf:ro ports: - "${MAIN_PORT}:10000" depends_on: - minio-service - couchdb-service + extra_hosts: + - "host.docker.internal:host-gateway" couchdb-service: container_name: budi-couchdb-dev diff --git a/hosting/kubernetes/nginx/Dockerfile b/hosting/kubernetes/nginx/Dockerfile new file mode 100644 index 000000000..189a71288 --- /dev/null +++ b/hosting/kubernetes/nginx/Dockerfile @@ -0,0 +1,3 @@ +# FROM envoyproxy/envoy:v1.16-latest +# COPY envoy.yaml /etc/envoy/envoy.yaml +# RUN chmod go+r /etc/envoy/envoy.yaml diff --git a/hosting/kubernetes/nginx/nginx.conf b/hosting/kubernetes/nginx/nginx.conf new file mode 100644 index 000000000..bab1f25c0 --- /dev/null +++ b/hosting/kubernetes/nginx/nginx.conf @@ -0,0 +1,146 @@ +static_resources: + listeners: + - name: main_listener + address: + socket_address: { address: 0.0.0.0, port_value: 10000 } + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: ingress + codec_type: auto + route_config: + name: local_route + virtual_hosts: + - name: local_services + domains: ["*"] + routes: + - match: { prefix: "/app/" } + route: + cluster: app-service + prefix_rewrite: "/" + + - match: { prefix: "/builder/" } + route: + cluster: app-service + + - match: { prefix: "/builder" } + route: + cluster: app-service + + - match: { prefix: "/app_" } + route: + cluster: app-service + + # special cases for worker admin (deprecated), global and system API + - match: { prefix: "/api/global/" } + route: + cluster: worker-service + + - match: { prefix: "/api/admin/" } + route: + cluster: worker-service + + - match: { prefix: "/api/system/" } + route: + cluster: worker-service + + - match: { path: "/" } + route: + cluster: app-service + + - match: + safe_regex: + google_re2: {} + regex: "/api/.*/export" + route: + timeout: 0s + cluster: app-service + + - match: { path: "/api/deploy" } + route: + timeout: 60s + cluster: app-service + + # special case for when API requests are made, can just forward, not to minio + - match: { prefix: "/api/" } + route: + cluster: app-service + + - match: { prefix: "/worker/" } + route: + cluster: worker-service + prefix_rewrite: "/" + + - match: { prefix: "/db/" } + route: + cluster: couchdb-service + prefix_rewrite: "/" + + # minio is on the default route because this works + # best, minio + AWS SDK doesn't handle path proxy + - match: { prefix: "/" } + route: + cluster: minio-service + + http_filters: + - name: envoy.filters.http.router + + clusters: + - name: app-service + connect_timeout: 0.25s + type: strict_dns + lb_policy: round_robin + load_assignment: + cluster_name: app-service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: app-service.budibase.svc.cluster.local + port_value: 4002 + + - name: minio-service + connect_timeout: 0.25s + type: strict_dns + lb_policy: round_robin + load_assignment: + cluster_name: minio-service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: minio-service.budibase.svc.cluster.local + port_value: 9000 + + - name: worker-service + connect_timeout: 0.25s + type: strict_dns + lb_policy: round_robin + load_assignment: + cluster_name: worker-service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: worker-service.budibase.svc.cluster.local + port_value: 4001 + + - name: couchdb-service + connect_timeout: 0.25s + type: strict_dns + lb_policy: round_robin + load_assignment: + cluster_name: couchdb-service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: budibase-prod-svc-couchdb + port_value: 5984 + diff --git a/hosting/nginx.dev.conf.hbs b/hosting/nginx.dev.conf.hbs new file mode 100644 index 000000000..f7e313495 --- /dev/null +++ b/hosting/nginx.dev.conf.hbs @@ -0,0 +1,91 @@ +user nginx; +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; +worker_processes auto; +worker_rlimit_nofile 33282; + +events { + worker_connections 4096; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + map $http_upgrade $connection_upgrade { + default "upgrade"; + } + + server { + listen 0.0.0.0:10000 default_server; + server_name _; + resolver 127.0.0.11 valid=10s; + + # security headers + add_header X-Frame-Options SAMEORIGIN always; + + # proxy headers + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + + # budibase services + set $app_service http://{{ address }}:4001; + set $worker_service http://{{ address }}:4002; + set $builder http://{{ address }}:3000; + set $minio http://minio-service:9000; + set $couchdb http://couchdb-service:5984; + + location = / { + return 301 /builder/; + } + + location /db/ { + proxy_pass $couchdb; + rewrite ^/db/(.*)$ /$1 break; + } + + location ~ ^/(global|admin|system)/ { + proxy_pass $worker_service; + } + + location /api/ { + proxy_read_timeout 120s; + proxy_connect_timeout 120s; + proxy_send_timeout 120s; + proxy_pass $app_service; + } + + location ^/(app_|builder) { + proxy_pass $app_service; + } + + location /app/ { + proxy_pass $app_service; + rewrite ^/app/(.*)$ /$1 break; + } + + {{!-- location = / { + proxy_pass $builder; + } --}} + + location /builder { + proxy_pass $builder; + rewrite ^/builder(.*)$ /builder/$1 break; + } + + location /builder/ { + proxy_pass $builder; + } + + location / { + proxy_pass $minio; + } + } +} \ No newline at end of file diff --git a/hosting/proxy/Dockerfile b/hosting/proxy/Dockerfile new file mode 100644 index 000000000..2fd64a9d6 --- /dev/null +++ b/hosting/proxy/Dockerfile @@ -0,0 +1,2 @@ +FROM nginx:latest +COPY nginx.conf /etc/nginx/nginx.conf \ No newline at end of file diff --git a/hosting/proxy/nginx.conf b/hosting/proxy/nginx.conf new file mode 100644 index 000000000..02480e83b --- /dev/null +++ b/hosting/proxy/nginx.conf @@ -0,0 +1,87 @@ +user nginx; +worker_processes 5; +error_log logs/error.log; +pid logs/nginx.pid; +worker_rlimit_nofile 8192; + +events { + worker_connections 4096; +} + +http { + include /etc/nginx/mime.types; + include /etc/nginx/proxy.conf; + # include /etc/nginx/fastcgi.conf; + # index index.html index.htm index.php; + + default_type application/octet-stream; + log_format main '$remote_addr - $remote_user [$time_local] $status ' + '"$request" $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + access_log logs/access.log main; + + server { + listen 10000 default_server; + listen [::]:10000 default_server; + server_name _; + access_log logs/budiproxy.access.log main; + + # security headers + add_header X-Frame-Options SAMEORIGIN always; + + upstream app-service { + server app-service:4002 + } + + upstream worker-service { + server worker-service:4003 + } + + location = /v1/update { + proxy_pass http://watchtower-service:8080; + } + + location /app/ { + proxy_pass http://app-service; + } + + # location ^/(app_|builder) { + # proxy_pass http://app-service:4002; + # } + + location /app_ { + proxy_pass http://app-service; + } + + location /builder { + proxy_pass http://app-service; + } + + location ~ ^/(global|admin|system)/ { + proxy_pass http://worker-service; + } + + location = / { + proxy_pass http://app-service; + } + + location /api/ { + proxy_read_timeout 120s; + proxy_connect_timeout 120s; + proxy_send_timeout 120s; + proxy_pass http://app-service; + } + + location /worker/ { + proxy_pass http://app-service; + } + + location /db/ { + proxy_pass http://couchdb-service:5984; + } + + location / { + proxy_pass http://minio-service:9000; + } + } +} \ No newline at end of file diff --git a/packages/server/scripts/dev/manage.js b/packages/server/scripts/dev/manage.js index f3938ad8e..1e1a11449 100644 --- a/packages/server/scripts/dev/manage.js +++ b/packages/server/scripts/dev/manage.js @@ -24,8 +24,8 @@ const Commands = { async function init() { // generate envoy file, always do this incase it has changed const hostingPath = path.join(process.cwd(), "..", "..", "hosting") - const envoyHbsPath = path.join(hostingPath, "envoy.dev.yaml.hbs") - const envoyOutputPath = path.join(hostingPath, ".generated-envoy.dev.yaml") + const envoyHbsPath = path.join(hostingPath, "nginx.dev.conf.hbs") + const envoyOutputPath = path.join(hostingPath, ".generated-nginx.dev.conf") const contents = fs.readFileSync(envoyHbsPath, "utf8") const config = { address: isLinux() ? "172.17.0.1" : "host.docker.internal",