diff --git a/.all-contributorsrc b/.all-contributorsrc index 53705907c..3a416f917 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -162,6 +162,7 @@ "translation" ] }, + { "login": "mslourens", "name": "Maurits Lourens", "avatar_url": "https://avatars.githubusercontent.com/u/1907152?v=4", diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index b4f773929..457d2c145 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -31,6 +31,9 @@ A clear and concise description of what you expected to happen. **Screenshots** If applicable, add screenshots to help explain your problem. + +**App Export** +If possible - please attach an export of your budibase application for debugging/reproduction purposes. **Desktop (please complete the following information):** - OS: [e.g. iOS] diff --git a/.github/workflows/README.md b/.github/workflows/README.md index c33665c96..f77323d85 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -119,6 +119,8 @@ This job is responsible for deploying to our production, cloud kubernetes enviro ## Pro +| **NOTE**: When developing for both pro / budibase repositories, your branch names need to match, or else the correct pro doesn't get run within your CI job. + ### Installing Pro The pro package is always installed from source in our CI jobs. @@ -132,7 +134,7 @@ This is done to prevent pro needing to be published prior to CI runs in budiabse - backend-core lives in the monorepo, so it can't be released independently to be used in pro - therefore the only option is to pull pro from source and release it as a part of the monorepo release, as if it were a mono package -The install is performed using the same steps as local development, via the `yarn bootstrap` command, see the [Contributing Guide#Pro](../CONTRIBUTING.md#pro) +The install is performed using the same steps as local development, via the `yarn bootstrap` command, see the [Contributing Guide#Pro](../../docs/CONTRIBUTING.md#pro) The branch to install pro from can vary depending on ref of the commit that triggered the budibase CI job. This is done to enable branches which have changes in both the monorepo and the pro repo to have their CI pass successfully. diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 1303d5921..e940e6fa1 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -7,7 +7,6 @@ on: branches: - master - develop - - new-design-ui pull_request: branches: - master @@ -60,19 +59,3 @@ jobs: with: install: false command: yarn test:e2e:ci - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: eu-west-1 - - - name: Upload to S3 - if: github.ref == 'refs/heads/new-design-ui' - run: | - tar -czvf new_ui.tar.gz packages/server/assets packages/server/index.html - aws s3 cp new_ui.tar.gz s3://prod-budi-app-assets/beta:design_ui/ - aws s3 cp packages/client/dist/budibase-client.js s3://prod-budi-app-assets/beta:design_ui/budibase-client.js - aws cloudfront create-invalidation --distribution-id E3ELKP4RCEHVLW --paths "/beta:design_ui/*" - diff --git a/.github/workflows/deploy-single-image.yml b/.github/workflows/deploy-single-image.yml new file mode 100644 index 000000000..8bf8f232c --- /dev/null +++ b/.github/workflows/deploy-single-image.yml @@ -0,0 +1,68 @@ +name: Deploy Budibase Single Container Image to DockerHub + +on: + workflow_dispatch: + +env: + BASE_BRANCH: ${{ github.event.pull_request.base.ref}} + BRANCH: ${{ github.event.pull_request.head.ref }} + CI: true + PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }} + REGISTRY_URL: registry.hub.docker.com +jobs: + build: + name: "build" + runs-on: ubuntu-latest + strategy: + matrix: + node-version: [14.x] + steps: + - name: "Checkout" + uses: actions/checkout@v2 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + - name: Setup QEMU + uses: docker/setup-qemu-action@v1 + - name: Setup Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + - name: Install Pro + run: yarn install:pro $BRANCH $BASE_BRANCH + - name: Run Yarn + run: yarn + - name: Run Yarn Bootstrap + run: yarn bootstrap + - name: Runt Yarn Lint + run: yarn lint + - name: Run Yarn Build + run: yarn build:docker:pre + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_API_KEY }} + - name: Get the latest release version + id: version + run: | + release_version=$(cat lerna.json | jq -r '.version') + echo $release_version + echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV + - name: Tag and release Budibase service docker image + uses: docker/build-push-action@v2 + with: + context: . + push: true + platforms: linux/amd64,linux/arm64 + tags: budibase/budibase,budibase/budibase:v${{ env.RELEASE_VERSION }} + file: ./hosting/single/Dockerfile + - name: Tag and release Budibase Azure App Service docker image + uses: docker/build-push-action@v2 + with: + context: . + push: true + platforms: linux/amd64 + build-args: TARGETBUILD=aas + tags: budibase/budibase-aas,budibase/budibase-aas:v${{ env.RELEASE_VERSION }} + file: ./hosting/single/Dockerfile diff --git a/.github/workflows/release-develop.yml b/.github/workflows/release-develop.yml index 631308d94..772fcf933 100644 --- a/.github/workflows/release-develop.yml +++ b/.github/workflows/release-develop.yml @@ -18,8 +18,9 @@ on: workflow_dispatch: env: - # Posthog token used by ui at build time - POSTHOG_TOKEN: phc_uDYOfnFt6wAbBAXkC6STjcrTpAFiWIhqgFcsC1UVO5F + # Posthog token used by ui at build time + # disable unless needed for testing + # POSTHOG_TOKEN: phc_uDYOfnFt6wAbBAXkC6STjcrTpAFiWIhqgFcsC1UVO5F INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }} PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} FEATURE_PREVIEW_URL: https://budirelease.live diff --git a/.github/workflows/release-selfhost.yml b/.github/workflows/release-selfhost.yml index fc2b7b0cc..da064f3e3 100644 --- a/.github/workflows/release-selfhost.yml +++ b/.github/workflows/release-selfhost.yml @@ -3,24 +3,37 @@ name: Budibase Release Selfhost on: workflow_dispatch: +env: + BRANCH: ${{ github.event.pull_request.head.ref }} + BASE_BRANCH: ${{ github.event.pull_request.base.ref}} + jobs: release: runs-on: ubuntu-latest steps: + - name: Fail if branch is not master + if: github.ref != 'refs/heads/master' + run: | + echo "Ref is not master, you must run this job from master." + exit 1 + - uses: actions/checkout@v2 with: node-version: 14.x fetch_depth: 0 + - name: Get the latest budibase release version + id: version + run: | + release_version=$(cat lerna.json | jq -r '.version') + echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV + - name: Tag and release Docker images (Self Host) run: | docker login -u $DOCKER_USER -p $DOCKER_PASSWORD - # Get latest release version - release_version=$(cat lerna.json | jq -r '.version') - echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - release_tag=v$release_version + release_tag=v${{ env.RELEASE_VERSION }} # Pull apps and worker images docker pull budibase/apps:$release_tag @@ -40,13 +53,15 @@ jobs: DOCKER_USER: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }} SELFHOST_TAG: latest - - - name: Build CLI executables + + - name: Install Pro + run: yarn install:pro $BRANCH $BASE_BRANCH + + - name: Bootstrap and build (CLI) run: | - pushd packages/cli yarn + yarn bootstrap yarn build - popd - name: Build OpenAPI spec run: | @@ -93,4 +108,4 @@ jobs: with: webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }} content: "Self Host Deployment Complete: ${{ env.RELEASE_VERSION }} deployed to Self Host." - embed-title: ${{ env.RELEASE_VERSION }} \ No newline at end of file + embed-title: ${{ env.RELEASE_VERSION }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f33dcc6d5..961082e1e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,10 +16,20 @@ on: - 'package.json' - 'yarn.lock' workflow_dispatch: + inputs: + versioning: + type: choice + description: "Versioning type: patch, minor, major" + default: patch + options: + - patch + - minor + - major + required: true env: # Posthog token used by ui at build time - POSTHOG_TOKEN: phc_fg5I3nDOf6oJVMHSaycEhpPdlgS8rzXG2r6F2IpxCHS + POSTHOG_TOKEN: phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }} SENTRY_DSN: ${{ secrets.SENTRY_DSN }} PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }} @@ -58,6 +68,7 @@ jobs: - name: Publish budibase packages to NPM env: NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + RELEASE_VERSION_TYPE: ${{ github.event.inputs.versioning }} run: | # setup the username and email. I tend to use 'GitHub Actions Bot' with no email by default git config --global user.name "Budibase Release Bot" diff --git a/.vscode/settings.json b/.vscode/settings.json index d471924fe..4838a4fd8 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,5 +3,17 @@ "editor.codeActionsOnSave": { "source.fixAll": true }, - "editor.defaultFormatter": "svelte.svelte-vscode" + "editor.defaultFormatter": "svelte.svelte-vscode", + "[json]": { + "editor.defaultFormatter": "vscode.json-language-features" + }, + "[javascript]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" + }, + "debug.javascript.terminalOptions": { + "skipFiles": [ + "${workspaceFolder}/packages/backend-core/node_modules/**", + "/**" + ] + }, } diff --git a/.yarnrc b/.yarnrc new file mode 100644 index 000000000..21fa517e2 --- /dev/null +++ b/.yarnrc @@ -0,0 +1 @@ +network-timeout 100000 diff --git a/README.md b/README.md index e8c6475d9..1dec1737d 100644 --- a/README.md +++ b/README.md @@ -135,13 +135,18 @@ You can learn more about the Budibase API at the following places: ## 🏁 Get started - - Deploy Budibase self-hosted in your existing infrastructure, using Docker, Kubernetes, and Digital Ocean. Or use Budibase Cloud if you don't need to self-host, and would like to get started quickly. ### [Get started with self-hosting Budibase](https://docs.budibase.com/docs/hosting-methods) +- [Docker - single ARM compatible image](https://docs.budibase.com/docs/docker) +- [Docker Compose](https://docs.budibase.com/docs/docker-compose) +- [Kubernetes](https://docs.budibase.com/docs/kubernetes-k8s) +- [Digital Ocean](https://docs.budibase.com/docs/digitalocean) +- [Portainer](https://docs.budibase.com/docs/portainer) + + ### [Get started with Budibase Cloud](https://budibase.com) @@ -164,7 +169,7 @@ If you have a question or would like to talk with other Budibase users and join ## ❗ Code of conduct -Budibase is dedicated to providing a welcoming, diverse, and harrassment-free experience for everyone. We expect everyone in the Budibase community to abide by our [**Code of Conduct**](https://github.com/Budibase/budibase/blob/HEAD/.github/CODE_OF_CONDUCT.md). Please read it. +Budibase is dedicated to providing a welcoming, diverse, and harrassment-free experience for everyone. We expect everyone in the Budibase community to abide by our [**Code of Conduct**](https://github.com/Budibase/budibase/blob/HEAD/docs/CODE_OF_CONDUCT.md). Please read it.
diff --git a/charts/budibase/Chart.yaml b/charts/budibase/Chart.yaml index 227a51543..570aa04d8 100644 --- a/charts/budibase/Chart.yaml +++ b/charts/budibase/Chart.yaml @@ -11,8 +11,8 @@ sources: - https://github.com/Budibase/budibase - https://budibase.com type: application -version: 0.2.10 -appVersion: 1.0.48 +version: 0.2.11 +appVersion: 1.0.214 dependencies: - name: couchdb version: 3.6.1 diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index ddc725d30..fd46e7764 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -122,6 +122,14 @@ spec: value: {{ .Values.globals.automationMaxIterations | quote }} - name: TENANT_FEATURE_FLAGS value: {{ .Values.globals.tenantFeatureFlags | quote }} + {{ if .Values.globals.bbAdminUserEmail }} + - name: BB_ADMIN_USER_EMAIL + value: { { .Values.globals.bbAdminUserEmail | quote } } + {{ end }} + {{ if .Values.globals.bbAdminUserPassword }} + - name: BB_ADMIN_USER_PASSWORD + value: { { .Values.globals.bbAdminUserPassword | quote } } + {{ end }} image: budibase/apps:{{ .Values.globals.appVersion }} imagePullPolicy: Always @@ -143,6 +151,10 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} + {{ if .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml .Values.imagePullSecrets | nindent 6 }} + {{ end }} restartPolicy: Always serviceAccountName: "" status: {} diff --git a/charts/budibase/templates/minio-service-deployment.yaml b/charts/budibase/templates/minio-service-deployment.yaml index 901fb61ad..103f9e3ed 100644 --- a/charts/budibase/templates/minio-service-deployment.yaml +++ b/charts/budibase/templates/minio-service-deployment.yaml @@ -68,6 +68,10 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} + {{ if .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml .Values.imagePullSecrets | nindent 6 }} + {{ end }} restartPolicy: Always serviceAccountName: "" volumes: @@ -75,4 +79,4 @@ spec: persistentVolumeClaim: claimName: minio-data status: {} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/charts/budibase/templates/proxy-service-deployment.yaml b/charts/budibase/templates/proxy-service-deployment.yaml index bd6a5e311..505a46f1e 100644 --- a/charts/budibase/templates/proxy-service-deployment.yaml +++ b/charts/budibase/templates/proxy-service-deployment.yaml @@ -40,6 +40,10 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} + {{ if .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml .Values.imagePullSecrets | nindent 6 }} + {{ end }} restartPolicy: Always serviceAccountName: "" volumes: diff --git a/charts/budibase/templates/redis-service-deployment.yaml b/charts/budibase/templates/redis-service-deployment.yaml index 0b6cb1256..6e09346ca 100644 --- a/charts/budibase/templates/redis-service-deployment.yaml +++ b/charts/budibase/templates/redis-service-deployment.yaml @@ -47,6 +47,10 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} + {{ if .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml .Values.imagePullSecrets | nindent 6 }} + {{ end }} restartPolicy: Always serviceAccountName: "" volumes: @@ -54,4 +58,4 @@ spec: persistentVolumeClaim: claimName: redis-data status: {} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index a7f05f313..918dab427 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -145,6 +145,10 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} + {{ if .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml .Values.imagePullSecrets | nindent 6 }} + {{ end }} restartPolicy: Always serviceAccountName: "" status: {} diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 2734202ff..404e92c70 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -91,7 +91,7 @@ globals: budibaseEnv: PRODUCTION enableAnalytics: "1" sentryDSN: "" - posthogToken: "phc_fg5I3nDOf6oJVMHSaycEhpPdlgS8rzXG2r6F2IpxCHS" + posthogToken: "phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU" logLevel: info selfHosted: "1" # set to 0 for budibase cloud environment, set to 1 for self-hosted setup multiTenancy: "0" # set to 0 to disable multiple orgs, set to 1 to enable multiple orgs diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 531ed0574..fb0848596 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -4,10 +4,10 @@ From opening a bug report to creating a pull request: every contribution is appr ## Table of contents -- [Quick start](#quick-start) -- [Status](#status) -- [What's included](#whats-included) -- [Bugs and feature requests](#bugs-and-feature-requests) +- [Where to start](#not-sure-where-to-start) +- [Contributor Licence Agreement](#contributor-license-agreement-cla) +- [Glossary of Terms](#glossary-of-terms) +- [Contributing to Budibase](#contributing-to-budibase) ## Not Sure Where to Start? @@ -32,6 +32,9 @@ All contributors must sign an [Individual Contributor License Agreement](https:/ If contributing on behalf of your company, your company must sign a [Corporate Contributor License Agreement](https://github.com/budibase/budibase/blob/next/.github/cla/corporate-cla.md). If so, please contact us via community@budibase.com. +If for any reason, your first contribution is in a PR created by other contributor, please just add a comment to the PR +with the following text to agree our CLA: "I have read the CLA Document and I hereby sign the CLA". + ## Glossary of Terms To understand the budibase API, it can be helpful to understand the top level entities that make up Budibase. @@ -162,7 +165,10 @@ When you are running locally, budibase stores data on disk using docker volumes. ### Development Modes -A combination of environment variables controls the mode budibase runs in. +A combination of environment variables controls the mode budibase runs in. + +| **NOTE**: You need to clean your browser cookies when you change between different modes. + Yarn commands can be used to mimic the different modes as described in the sections below: #### Self Hosted @@ -189,7 +195,7 @@ To enable this mode, use: yarn mode:account ``` ### CI - An overview of the CI pipelines can be found [here](./workflows/README.md) + An overview of the CI pipelines can be found [here](../.github/workflows/README.md) ### Pro diff --git a/docs/DEV-SETUP-MACOSX.md b/docs/DEV-SETUP-MACOSX.md index 5606fd0d1..c5990e58d 100644 --- a/docs/DEV-SETUP-MACOSX.md +++ b/docs/DEV-SETUP-MACOSX.md @@ -4,6 +4,11 @@ Install instructions [here](https://brew.sh/) +| **NOTE**: If you are working on a M1 Apple Silicon which is running Z shell, you could need to add +`eval $(/opt/homebrew/bin/brew shellenv)` line to your `.zshrc`. This will make your zsh to find the apps you install +through brew. + + ### Install Node Budibase requires a recent version of node (14+): @@ -51,4 +56,7 @@ So this command will actually run the application in dev mode. It creates .env f The dev version will be available on port 10000 i.e. -http://127.0.0.1:10000/builder/admin \ No newline at end of file +http://127.0.0.1:10000/builder/admin + +| **NOTE**: If you are working on a M1 Apple Silicon, you will need to uncomment `# platform: linux/amd64` line in +[hosting/docker-compose-dev.yaml](../hosting/docker-compose.dev.yaml) \ No newline at end of file diff --git a/hosting/.env b/hosting/.env index 39df76d01..11dd661bf 100644 --- a/hosting/.env +++ b/hosting/.env @@ -18,4 +18,8 @@ MINIO_PORT=4004 COUCH_DB_PORT=4005 REDIS_PORT=6379 WATCHTOWER_PORT=6161 -BUDIBASE_ENVIRONMENT=PRODUCTION \ No newline at end of file +BUDIBASE_ENVIRONMENT=PRODUCTION + +# An admin user can be automatically created initially if these are set +BB_ADMIN_USER_EMAIL= +BB_ADMIN_USER_PASSWORD= \ No newline at end of file diff --git a/hosting/docker-compose.dev.yaml b/hosting/docker-compose.dev.yaml index be0bc74a2..7322b0e8a 100644 --- a/hosting/docker-compose.dev.yaml +++ b/hosting/docker-compose.dev.yaml @@ -11,10 +11,11 @@ services: - minio_data:/data ports: - "${MINIO_PORT}:9000" + - "9001:9001" environment: MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} - command: server /data + command: server /data --console-address ":9001" healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] interval: 30s diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index f9d9eaf1c..7d3e6960d 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -23,6 +23,8 @@ services: ENABLE_ANALYTICS: "true" REDIS_URL: redis-service:6379 REDIS_PASSWORD: ${REDIS_PASSWORD} + BB_ADMIN_USER_EMAIL: ${BB_ADMIN_USER_EMAIL} + BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD} depends_on: - worker-service - redis-service @@ -61,7 +63,7 @@ services: MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} MINIO_BROWSER: "off" - command: server /data + command: server /data --console-address ":9001" healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] interval: 30s @@ -74,6 +76,8 @@ services: - "${MAIN_PORT}:10000" container_name: bbproxy image: budibase/proxy + environment: + - PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10 depends_on: - minio-service - worker-service diff --git a/hosting/hosting.properties b/hosting/hosting.properties index c8e2f5c60..11dd661bf 100644 --- a/hosting/hosting.properties +++ b/hosting/hosting.properties @@ -19,3 +19,7 @@ COUCH_DB_PORT=4005 REDIS_PORT=6379 WATCHTOWER_PORT=6161 BUDIBASE_ENVIRONMENT=PRODUCTION + +# An admin user can be automatically created initially if these are set +BB_ADMIN_USER_EMAIL= +BB_ADMIN_USER_PASSWORD= \ No newline at end of file diff --git a/hosting/nginx.prod.conf.hbs b/hosting/nginx.prod.conf.hbs index ac35a2020..421362630 100644 --- a/hosting/nginx.prod.conf.hbs +++ b/hosting/nginx.prod.conf.hbs @@ -9,7 +9,11 @@ events { } http { + # rate limiting + limit_req_status 429; limit_req_zone $binary_remote_addr zone=ratelimit:10m rate=20r/s; + limit_req_zone $binary_remote_addr zone=webhooks:10m rate=${PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND}r/s; + include /etc/nginx/mime.types; default_type application/octet-stream; proxy_set_header Host $host; @@ -126,6 +130,25 @@ http { proxy_pass http://$apps:4002; } + location /api/webhooks/ { + # calls to webhooks are rate limited + limit_req zone=webhooks nodelay; + + # Rest of configuration copied from /api/ location above + # 120s timeout on API requests + proxy_read_timeout 120s; + proxy_connect_timeout 120s; + proxy_send_timeout 120s; + + proxy_http_version 1.1; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + proxy_pass http://$apps:4002; + } + location /db/ { proxy_pass http://$couchdb:5984; rewrite ^/db/(.*)$ /$1 break; diff --git a/hosting/proxy/Dockerfile b/hosting/proxy/Dockerfile index a2b17d333..d9b33e3e9 100644 --- a/hosting/proxy/Dockerfile +++ b/hosting/proxy/Dockerfile @@ -1,3 +1,13 @@ FROM nginx:latest -COPY .generated-nginx.prod.conf /etc/nginx/nginx.conf -COPY error.html /usr/share/nginx/html/error.html \ No newline at end of file + +# nginx.conf +# use the default nginx behaviour for *.template files which are processed with envsubst +# override the output dir to output directly to /etc/nginx instead of /etc/nginx/conf.d +ENV NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx +COPY .generated-nginx.prod.conf /etc/nginx/templates/nginx.conf.template + +# Error handling +COPY error.html /usr/share/nginx/html/error.html + +# Default environment +ENV PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10 \ No newline at end of file diff --git a/hosting/scripts/build-target-paths.sh b/hosting/scripts/build-target-paths.sh index d1c9b5cd0..6d5a8429f 100644 --- a/hosting/scripts/build-target-paths.sh +++ b/hosting/scripts/build-target-paths.sh @@ -3,15 +3,14 @@ echo ${TARGETBUILD} > /buildtarget.txt if [[ "${TARGETBUILD}" = "aas" ]]; then # Azure AppService uses /home for persisent data & SSH on port 2222 - mkdir -p /home/budibase/{minio,couchdb} - mkdir -p /home/budibase/couchdb/data - chown -R couchdb:couchdb /home/budibase/couchdb/ + DATA_DIR=/home + mkdir -p $DATA_DIR/{search,minio,couchdb} + mkdir -p $DATA_DIR/couchdb/{dbs,views} + chown -R couchdb:couchdb $DATA_DIR/couchdb/ apt update apt-get install -y openssh-server - sed -i 's#dir=/opt/couchdb/data/search#dir=/home/budibase/couchdb/data/search#' /opt/clouseau/clouseau.ini - sed -i 's#/minio/minio server /minio &#/minio/minio server /home/budibase/minio &#' /runner.sh - sed -i 's#database_dir = ./data#database_dir = /home/budibase/couchdb/data#' /opt/couchdb/etc/default.ini - sed -i 's#view_index_dir = ./data#view_index_dir = /home/budibase/couchdb/data#' /opt/couchdb/etc/default.ini sed -i "s/#Port 22/Port 2222/" /etc/ssh/sshd_config /etc/init.d/ssh restart fi + +sed -i 's#DATA_DIR#$DATA_DIR#' /opt/clouseau/clouseau.ini /opt/couchdb/etc/local.ini diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index 5e1b0b137..476a6e5e9 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -20,10 +20,10 @@ RUN node /pinVersions.js && yarn && yarn build && /cleanup.sh FROM couchdb:3.2.1 # TARGETARCH can be amd64 or arm e.g. docker build --build-arg TARGETARCH=amd64 -ARG TARGETARCH amd64 +ARG TARGETARCH=amd64 #TARGETBUILD can be set to single (for single docker image) or aas (for azure app service) # e.g. docker build --build-arg TARGETBUILD=aas .... -ARG TARGETBUILD single +ARG TARGETBUILD=single ENV TARGETBUILD $TARGETBUILD COPY --from=build /app /app @@ -34,27 +34,33 @@ ENV \ ARCHITECTURE=amd \ BUDIBASE_ENVIRONMENT=PRODUCTION \ CLUSTER_PORT=80 \ - COUCHDB_PASSWORD=budibase \ - COUCHDB_USER=budibase \ - COUCH_DB_URL=http://budibase:budibase@localhost:5984 \ # CUSTOM_DOMAIN=budi001.custom.com \ + DATA_DIR=/data \ DEPLOYMENT_ENVIRONMENT=docker \ - INTERNAL_API_KEY=budibase \ - JWT_SECRET=testsecret \ - MINIO_ACCESS_KEY=budibase \ - MINIO_SECRET_KEY=budibase \ MINIO_URL=http://localhost:9000 \ - POSTHOG_TOKEN=phc_fg5I3nDOf6oJVMHSaycEhpPdlgS8rzXG2r6F2IpxCHS \ - REDIS_PASSWORD=budibase \ + POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU \ REDIS_URL=localhost:6379 \ SELF_HOSTED=1 \ TARGETBUILD=$TARGETBUILD \ WORKER_PORT=4002 \ - WORKER_URL=http://localhost:4002 + WORKER_URL=http://localhost:4002 \ + APPS_URL=http://localhost:4001 + +# These secret env variables are generated by the runner at startup +# their values can be overriden by the user, they will be written +# to the .env file in the /data directory for use later on +# REDIS_PASSWORD=budibase \ +# COUCHDB_PASSWORD=budibase \ +# COUCHDB_USER=budibase \ +# COUCH_DB_URL=http://budibase:budibase@localhost:5984 \ +# INTERNAL_API_KEY=budibase \ +# JWT_SECRET=testsecret \ +# MINIO_ACCESS_KEY=budibase \ +# MINIO_SECRET_KEY=budibase \ # install base dependencies RUN apt-get update && \ - apt-get install -y software-properties-common wget nginx && \ + apt-get install -y software-properties-common wget nginx uuid-runtime && \ apt-add-repository 'deb http://security.debian.org/debian-security stretch/updates main' && \ apt-get update @@ -66,8 +72,8 @@ RUN curl -sL https://deb.nodesource.com/setup_16.x -o /tmp/nodesource_setup.sh & npm install --global yarn pm2 # setup nginx -ADD hosting/single/nginx.conf /etc/nginx -ADD hosting/single/nginx-default-site.conf /etc/nginx/sites-enabled/default +ADD hosting/single/nginx/nginx.conf /etc/nginx +ADD hosting/single/nginx/nginx-default-site.conf /etc/nginx/sites-enabled/default RUN mkdir -p /var/log/nginx && \ touch /var/log/nginx/error.log && \ touch /var/run/nginx.pid @@ -86,13 +92,13 @@ RUN wget https://github.com/cloudant-labs/clouseau/releases/download/2.21.0/clou WORKDIR /opt/clouseau RUN mkdir ./bin -ADD hosting/single/clouseau ./bin/ -ADD hosting/single/log4j.properties hosting/single/clouseau.ini ./ +ADD hosting/single/clouseau/clouseau ./bin/ +ADD hosting/single/clouseau/log4j.properties hosting/single/clouseau/clouseau.ini ./ RUN chmod +x ./bin/clouseau # setup CouchDB WORKDIR /opt/couchdb -ADD hosting/single/vm.args ./etc/ +ADD hosting/single/couch/vm.args hosting/single/couch/local.ini ./etc/ # setup minio WORKDIR /minio @@ -103,12 +109,13 @@ RUN chmod +x install.sh && ./install.sh WORKDIR / ADD hosting/single/runner.sh . RUN chmod +x ./runner.sh -ADD hosting/scripts/healthcheck.sh . +ADD hosting/single/healthcheck.sh . RUN chmod +x ./healthcheck.sh ADD hosting/scripts/build-target-paths.sh . RUN chmod +x ./build-target-paths.sh +# Script below sets the path for storing data based on $DATA_DIR # For Azure App Service install SSH & point data locations to /home RUN /build-target-paths.sh @@ -117,8 +124,7 @@ RUN yarn cache clean -f EXPOSE 80 EXPOSE 443 -VOLUME /opt/couchdb/data -VOLUME /minio +VOLUME /data # setup letsencrypt certificate RUN apt-get install -y certbot python3-certbot-nginx diff --git a/hosting/single/clouseau b/hosting/single/clouseau/clouseau similarity index 100% rename from hosting/single/clouseau rename to hosting/single/clouseau/clouseau diff --git a/hosting/single/clouseau.ini b/hosting/single/clouseau/clouseau.ini similarity index 92% rename from hosting/single/clouseau.ini rename to hosting/single/clouseau/clouseau.ini index f086cf039..578a5acaf 100644 --- a/hosting/single/clouseau.ini +++ b/hosting/single/clouseau/clouseau.ini @@ -7,7 +7,7 @@ name=clouseau@127.0.0.1 cookie=monster ; the path where you would like to store the search index files -dir=/opt/couchdb/data/search +dir=DATA_DIR/search ; the number of search indexes that can be open simultaneously max_indexes_open=500 diff --git a/hosting/single/log4j.properties b/hosting/single/clouseau/log4j.properties similarity index 100% rename from hosting/single/log4j.properties rename to hosting/single/clouseau/log4j.properties diff --git a/hosting/single/couch/local.ini b/hosting/single/couch/local.ini new file mode 100644 index 000000000..266c0d4b6 --- /dev/null +++ b/hosting/single/couch/local.ini @@ -0,0 +1,5 @@ +; CouchDB Configuration Settings + +[couchdb] +database_dir = DATA_DIR/couch/dbs +view_index_dir = DATA_DIR/couch/views diff --git a/hosting/single/vm.args b/hosting/single/couch/vm.args similarity index 100% rename from hosting/single/vm.args rename to hosting/single/couch/vm.args diff --git a/hosting/scripts/healthcheck.sh b/hosting/single/healthcheck.sh similarity index 93% rename from hosting/scripts/healthcheck.sh rename to hosting/single/healthcheck.sh index 80f2ece0b..b92cd153a 100644 --- a/hosting/scripts/healthcheck.sh +++ b/hosting/single/healthcheck.sh @@ -1,6 +1,10 @@ #!/usr/bin/env bash healthy=true +if [ -f "/data/.env" ]; then + export $(cat /data/.env | xargs) +fi + if [[ $(curl -Lfk -s -w "%{http_code}\n" http://localhost/ -o /dev/null) -ne 200 ]]; then echo 'ERROR: Budibase is not running'; healthy=false diff --git a/hosting/single/nginx-default-site.conf b/hosting/single/nginx/nginx-default-site.conf similarity index 99% rename from hosting/single/nginx-default-site.conf rename to hosting/single/nginx/nginx-default-site.conf index 964313fa7..c0d80a018 100644 --- a/hosting/single/nginx-default-site.conf +++ b/hosting/single/nginx/nginx-default-site.conf @@ -88,7 +88,4 @@ server { gzip_proxied any; gzip_comp_level 6; gzip_types text/plain text/css text/xml application/json application/javascript application/rss+xml application/atom+xml image/svg+xml; - - - } diff --git a/hosting/single/nginx.conf b/hosting/single/nginx/nginx.conf similarity index 100% rename from hosting/single/nginx.conf rename to hosting/single/nginx/nginx.conf diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh index 6f3d24784..09387343b 100644 --- a/hosting/single/runner.sh +++ b/hosting/single/runner.sh @@ -1,6 +1,48 @@ +#!/bin/bash +declare -a ENV_VARS=("COUCHDB_USER" "COUCHDB_PASSWORD" "DATA_DIR" "MINIO_ACCESS_KEY" "MINIO_SECRET_KEY" "INTERNAL_API_KEY" "JWT_SECRET" "REDIS_PASSWORD") + +# Azure App Service customisations +if [[ "${TARGETBUILD}" = "aas" ]]; then + DATA_DIR=/home + /etc/init.d/ssh start +else + DATA_DIR=${DATA_DIR:-/data} +fi + +if [ -f "${DATA_DIR}/.env" ]; then + export $(cat ${DATA_DIR}/.env | xargs) +fi +# first randomise any unset environment variables +for ENV_VAR in "${ENV_VARS[@]}" +do + temp=$(eval "echo \$$ENV_VAR") + if [[ -z "${temp}" ]]; then + eval "export $ENV_VAR=$(uuidgen | sed -e 's/-//g')" + fi +done +if [[ -z "${COUCH_DB_URL}" ]]; then + export COUCH_DB_URL=http://$COUCHDB_USER:$COUCHDB_PASSWORD@localhost:5984 +fi +if [ ! -f "${DATA_DIR}/.env" ]; then + touch ${DATA_DIR}/.env + for ENV_VAR in "${ENV_VARS[@]}" + do + temp=$(eval "echo \$$ENV_VAR") + echo "$ENV_VAR=$temp" >> ${DATA_DIR}/.env + done + echo "COUCH_DB_URL=${COUCH_DB_URL}" >> ${DATA_DIR}/.env +fi + +export COUCH_DB_URL=http://$COUCHDB_USER:$COUCHDB_PASSWORD@localhost:5984 + +# make these directories in runner, incase of mount +mkdir -p ${DATA_DIR}/couchdb/{dbs,views} +mkdir -p ${DATA_DIR}/minio +mkdir -p ${DATA_DIR}/search +chown -R couchdb:couchdb ${DATA_DIR}/couchdb redis-server --requirepass $REDIS_PASSWORD & /opt/clouseau/bin/clouseau & -/minio/minio server /minio & +/minio/minio server ${DATA_DIR}/minio & /docker-entrypoint.sh /opt/couchdb/bin/couchdb & /etc/init.d/nginx restart if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then diff --git a/hosting/single/test.sh b/hosting/single/test.sh index c7ef53f99..8830426a4 100755 --- a/hosting/single/test.sh +++ b/hosting/single/test.sh @@ -1,4 +1,4 @@ #!/bin/bash -id=$(docker run -t -d -p 80:80 budibase:latest) +id=$(docker run -t -d -p 8080:80 budibase:latest) docker exec -it $id bash docker kill $id diff --git a/i18n/README.es.md b/i18n/README.es.md index 7245dc865..21eb8caef 100644 --- a/i18n/README.es.md +++ b/i18n/README.es.md @@ -8,10 +8,11 @@

- Construye herramientas empresariales personalizadas en cuestión de minutos y en su propia infraestructura. + Construye herramientas empresariales personalizadas en cuestión de minutos y en tu propia infraestructura.

- Budibase es una plataforma de código bajo de código abierto, que ayuda a desarrolladores y profesionales de TI a crear, automatizar y enviar aplicaciones empresariales personalizadas en cuestión de minutos y en su propia infraestructura + Budibase es una plataforma low code de código abierto, que ayuda a desarrolladores y profesionales de TI a crear y +automatizar aplicaciones personalizadas en cuestión de minutos

@@ -20,7 +21,7 @@

- + Budibase design ui

@@ -30,9 +31,6 @@ GitHub release (latest by date) - - Discord - Follow @budibase @@ -43,130 +41,213 @@

- Sign-up + Comenzar con Budibase en la nube + · + Comenzar con Docker, K8s, DO · - Docs + Documentaciones · - Feature request + Pedir una funcionalidad · - Report a bug + Reportar un error · - Support: Discussions - & - Discord + Support: Comunidad

+

+## ✨ Caracteristicas + +### Construir aplicaciones reales +Con Budibase podras construir aplicaciones de pagina unica de gran rendimiento. Ademas, puedes hacerlas con un diseño +adaptativo para darles a tus usuarios una gran experiencia. +

+ +### Codigo abierto y ampliable +Budibase es de codigo abierto con licencia GPL v3. Puedes ampliarlo o modificarlo para adaptarlo a tus necesidades y preferencias. -## ✨ Features -When other platforms chose the closed source route, we decided to go open source. When other platforms chose cloud builders, we decided a local builder offered the better developer experience. We like to do things differently at Budibase. +De esta manera proveemos una buena experiencia para el desarrollador asi como establecemos la confianza de que Budibase siempre estara funcional. +

-- **Build and ship real software.** Unlike other platforms, with Budibase you build and ship single page applications. Budibase applications have performance baked in and can be designed responsively, providing your users with a great experience. +### Cargar informacion o empezar desde cero +Budibase permite importar datos desde multiples fuentes, entre las que estan incluidas: MondoDB, CouchDB, PostgreSQL, MySQL, +Airtable, S3, DynamoDB o API REST. -- **Open source and extensable.** Budibase is open-source. The builder is licensed AGPL v3, the server is GPL v3, and the client is MPL. This should fill you with confidence that Budibase will always be around. You can also code against Budibase or fork it and make changes as you please, providing a developer-friendly experience. +O si lo prefieres, con Budibase puedes empezar desde cero y construir tus propias aplicaciones +sin necesidad de herramientas externas. +[Sugerir fuente de datos](https://github.com/Budibase/budibase/discussions?discussions_q=category%3AIdeas). -- **Load data or start from scratch.** Budibase pulls in data from multiple sources, including MongoDB, CouchDB, PostgreSQL, mySQL, Airtable, Google Sheets, S3, DyanmoDB, or a REST API. And unlike other platforms, with Budibase you can start from scratch and create business apps with no data sources. [Request new data sources](https://github.com/Budibase/budibase/discussions?discussions_q=category%3AIdeas). +

+ Budibase data +

+

-- **Design and build apps with powerful pre-made components.** Budibase comes out of the box with beautifully designed, powerful components which you can use like building blocks to build your UI. We also expose a lot of your favourite CSS styling options so you can go that extra creative mile. [Request new components](https://github.com/Budibase/budibase/discussions?discussions_q=category%3AIdeas). +### Diseña y construye aplicaciones con componentes profesionales prediseñados -- **Automate processes, integrate with other tools, and connect to webhooks.** Save time by automating manual processes and workflows. From connecting to webhooks, to automating emails, simply tell Budibase what to do and let it work for you. You can easily [create new automations for Budibase here](https://github.com/Budibase/automations) or [request new integrations here](https://github.com/Budibase/budibase/discussions?discussions_q=category%3AIdeas). +Budibase incorpora componentes profesionales prediseñados que podras usar de manera facil e intuitiva +como bloques de construccion para la interfaz de tu aplicacion. -- **Cloud hosting and self-hosting.** Users can self-host (see below), or host their apps with Budibase. Currently, our cloud hosting offering is limited to the free tier but we aim to change this in the future. For heavy usage, we advise users to self-host. +Tambien mostramos gran parte del CSS para que puedas adaptar los componentes a tus diseños. +[Sugerir componente](https://github.com/Budibase/budibase/discussions?discussions_q=category%3AIdeas).

- Budibase design ui + Budibase design

+

+ +### Procesos automatizados, integra tu aplicacion con otras herramientas y conectala a eventos webhook + +Ahorra tiempo automatizando flujos de trabajo y procesos manuales. Podras desde conectar eventos webhook hasta automatizar emails, +simplemente dile a Budibase que hacer y deja que el haga el trabajo por ti. +[Crear nuevos procesos automatizados](https://github.com/Budibase/automations) o [Sugerir proceso automatizado](https://github.com/Budibase/budibase/discussions?discussions_q=category%3AIdeas). +

+ Budibase automations +

+

-## ⌛ Status -- [x] Alpha: We are demoing Budibase to users and receiving feedback -- [x] Private Beta: We are testing Budibase with a closed set of customers -- [x] Public Beta: Anyone can [sign-up and use Budibase](https://portal.budi.live/signup). -- [ ] Official Launch +### Tus herramientas favoritas -Watch "releases" of this repo to get notified of major updates, and give the star button a click whilst you're there. +Budibase integra un gran numero de herramientas que te permitiran construir tus aplicaciones ajustandose a tus preferencias.

- + Budibase integrations

+

-### Stargazers over time +### Un paraiso para administradores -[![Stargazers over time](https://starchart.cc/Budibase/budibase.svg)](https://starchart.cc/Budibase/budibase) +Puedes albergar Budibase en tu propia infraestructura y gestionar globalmente usuarios, incorporaciones, SMTP, aplicaciones, +grupos, diseños de temas, etc. + +Tambien puedes gestionar los usuarios y grupos, o delegar en personas asignadas para ello, desde nuestra aplicacion sin +mucho esfuerzo. + +Budibase is made to scale. With Budibase, you can self-host on your own infrastructure and globally manage users, onboarding, SMTP, apps, groups, theming and more. You can also provide users/groups with an app portal and disseminate user-management to the group manager. + +- Video Promocional: https://youtu.be/xoljVpty_Kw + +
+ +--- -If you are having issues between updates of the builder, please use the guide [here](https://github.com/Budibase/budibase/blob/HEAD/.github/CONTRIBUTING.md#troubleshooting) to clear down your environment. +
-## 🏁 Getting Started with Budibase +## Budibase API Publica -The Budibase builder runs in Electron, on Mac, PC and Linux. Follow the steps below to get started: -- [ ] [Sign-up to Budibase](https://portal.budi.live/signup) -- [ ] Create a username and password -- [ ] Copy your API key -- [ ] Download Budibase -- [ ] Open Budibase and enter your API key +Como todo lo que construimos en Budibase, nuestra nueva API publica es facil de usar, flexible e introduce nueva ampliacion +del sistema. Budibase API ofrece: +- Uso de Budibase como backend +- Interoperabilidad -[Here is a guided tutorial](https://docs.budibase.com/tutorial/tutorial-signing-up) if you need extra help. +#### Documentacion +Puedes aprender mas acerca de Budibase API en los siguientes documentos: +- [Documentacion general](https://docs.budibase.com/docs/public-api) : Como optener tu clave para la API, usar Insomnia y Postman +- [API Interactiva](https://docs.budibase.com/reference/post_applications) : Aprende como trabajar con la API -## 🤖 Self-hosting +#### Guias -Budibase wants to make sure anyone can use the tools we develop and we know a lot of people need to be able to host the apps they make on their own systems - that is why we've decided to try and make self hosting as easy as possible! +- [Construye una aplicacion con Budibase y Next.js](https://budibase.com/blog/building-a-crud-app-with-budibase-and-next.js/) + +

+ Budibase data +

+

+ +


+ +## 🏁 Comenzar con Budibase + +Puedes alojar Budibase en tu propia infraestructura con Docker, Kubernetes o Digital Ocean; o usa Budibase en la nube si +quieres empezar a crear tus aplicaciones rapidamente y sin ningun tipo de preocupacion. + +### [Comenzar con Budibase self-hosting](https://docs.budibase.com/docs/hosting-methods) + +- [Docker - single ARM compatible image](https://docs.budibase.com/docs/docker) +- [Docker Compose](https://docs.budibase.com/docs/docker-compose) +- [Kubernetes](https://docs.budibase.com/docs/kubernetes-k8s) +- [Digital Ocean](https://docs.budibase.com/docs/digitalocean) +- [Portainer](https://docs.budibase.com/docs/portainer) -Currently, you can host your apps using Docker or Digital Ocean. The documentation for self-hosting can be found [here](https://docs.budibase.com/docs/hosting-methods). -[![Deploy to DO](https://www.deploytodo.com/do-btn-blue.svg)](https://cloud.digitalocean.com/droplets/new?onboarding_origin=marketplace&i=09038e&fleetUuid=bb04f9c8-1de8-4687-b2ae-1d5177a0535b&appId=77729671&type=applications&size=s-4vcpu-8gb®ion=nyc1&refcode=0caaa6085a82&image=budibase-20-04) +### [Comenzar con Budibase en la nube](https://budibase.com) +

-## 🎓 Learning Budibase +## 🎓 Aprende a usar Budibase + +Aqui tienes la [documentacion de Budibase](https://docs.budibase.com/docs). +
-The Budibase [documentation lives here](https://docs.budibase.com). -You can also follow a quick tutorial on [how to build a CRM with Budibase](https://docs.budibase.com/tutorial/tutorial-introduction) +

+## 💬 Comunidad -## Roadmap +Te invitamos a que te unas a nuestra comunidad de Budibase, alli podras hacer las preguntas que quieras, ayudar a otras +personas o tener una charla entretenida con otros usuarios de Budibase. +[Acceder a la comunidad de Budibase](https://github.com/Budibase/budibase/discussions) +


-Checkout our [Public Roadmap](https://github.com/Budibase/budibase/projects/10). If you would like to discuss some of the items on the roadmap, please feel to reach out on [Discord](https://discord.gg/rCYayfe), or via [Github discussions](https://github.com/Budibase/budibase/discussions) +## ❗ Codigo de conducta -## ❗ Code of Conduct +Budibase presta especial atencion en acoger a personas de toda diversidad y ofrecer un entorno de respeto mutuo. Asi mismo +esperamos lo mismo de nuestra comunidad, por favor lee el +[**Codigo de conducta**](https://github.com/Budibase/budibase/blob/HEAD/.github/CODE_OF_CONDUCT.md). +
-Budibase is dedicated to providing a welcoming, diverse, and harrassment-free experience for everyone. We expect everyone in the Budibase community to abide by our [**Code of Conduct**](https://github.com/Budibase/budibase/blob/HEAD/.github/CODE_OF_CONDUCT.md). Please read it. +

-## 🙌 Contributing to Budibase -From opening a bug report to creating a pull request: every contribution is appreciated and welcomed. If you're planning to implement a new feature or change the API please create an issue first. This way we can ensure your work is not in vain. +## 🙌 Contribuir en Budibase -### Not Sure Where to Start? -A good place to start contributing, is the [First time issues project](https://github.com/Budibase/budibase/projects/22). +Desde comunicar un bug a solventar un error en el codigo, toda contribucion es apreciada y bienvenida. Si estas planeando +implementar una nueva funcionalidad o un realizar un cambio en la API, por favor crea un [nuevo mensaje aqui](https://github.com/Budibase/budibase/issues), +de esta manera nos encargaremos que tu trabajo no sea en vano. -### How the repository is organized -Budibase is a monorepo managed by lerna. Lerna manages the building and publishing of the budibase packages. At a high level, here are the packages that make up Budibase. +Aqui tienes instrucciones de como configurar tu entorno Budibase para [Debian](https://github.com/Budibase/budibase/tree/HEAD/docs/DEV-SETUP-DEBIAN.md) +y [MacOSX](https://github.com/Budibase/budibase/tree/HEAD/docs/DEV-SETUP-MACOSX.md) -- [packages/builder](https://github.com/Budibase/budibase/tree/HEAD/packages/builder) - contains code for the budibase builder client side svelte application. +### No estas seguro por donde empezar? +Un buen lugar para empezar a contribuir con nosotros es [aqui](https://github.com/Budibase/budibase/projects/22). -- [packages/client](https://github.com/Budibase/budibase/tree/HEAD/packages/client) - A module that runs in the browser responsible for reading JSON definition and creating living, breathing web apps from it. +### Organizacion del repositorio -- [packages/server](https://github.com/Budibase/budibase/tree/HEAD/packages/server) - The budibase server. This Koa app is responsible for serving the JS for the builder and budibase apps, as well as providing the API for interaction with the database and file system. +Budibase es un repositorio unico gestionado por Lerna. Lerna construye y publica los paquetes de Budibase sincronizandolos +cada ves que se realiza un cambio. A rasgos generales, estos son los paquetes que conforman Budibase: -For more information, see [CONTRIBUTING.md](https://github.com/Budibase/budibase/blob/HEAD/.github/CONTRIBUTING.md) +- [packages/builder](https://github.com/Budibase/budibase/tree/HEAD/packages/builder) - contiene el codigo del builder de la parte cliente, esta es una aplicacion svelte. -## 📝 License +- [packages/client](https://github.com/Budibase/budibase/tree/HEAD/packages/client) - Este modulo se ejecuta en el browser y es el responsable de leer definiciones JSON y crear aplicaciones web en el momento. -Budibase is open-source. The builder is licensed [AGPL v3](https://www.gnu.org/licenses/agpl-3.0.en.html), the server is licensed [GPL v3](https://www.gnu.org/licenses/gpl-3.0.en.html), and the client is licensed [MPL](https://directory.fsf.org/wiki/License:MPL-2.0). +- [packages/server](https://github.com/Budibase/budibase/tree/HEAD/packages/server) - La parte servidor de Budibase. Esta aplicacion Koa es responsable de suministrar lo necesario al builder para asi generar las aplicaciones Budibase. Tambien provee una API para interaccionar con la base de datos y el almacenamiento de ficheros. -## 💬 Get in touch +Para mas informacion, por favor lee el siguiente documento [CONTRIBUTING.md](https://github.com/Budibase/budibase/blob/HEAD/docs/CONTRIBUTING.md) -If you have a question or would like to talk with other Budibase users, please hop over to [Github discussions](https://github.com/Budibase/budibase/discussions) or join our Discord server: +

-[Discord chatroom](https://discord.gg/rCYayfe) -![Discord Shield](https://discordapp.com/api/guilds/733030666647765003/widget.png?style=shield) +## 📝 Licencia +Budibase es open-source, licenciado como [GPL v3](https://www.gnu.org/licenses/gpl-3.0.en.html). El cliente y las librerias +de componentes estan licenciadas como [MPL](https://directory.fsf.org/wiki/License:MPL-2.0) - de esta manera, puedes licenciar +como tu quieras las aplicaciones que construyas. -## Contributors ✨ +

-Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)): +## ⭐ Historia de nuestros Stargazers + +[![Stargazers over time](https://starchart.cc/Budibase/budibase.svg)](https://starchart.cc/Budibase/budibase) + +Si estas teniendo problemas con el builder despues de actualizar, por favor [lee esta guia](https://github.com/Budibase/budibase/blob/HEAD/docs/CONTRIBUTING.md#troubleshooting) to clear down your environment. + +

+ +## Contribuidores ✨ + +Queremos prestar un especial agradecimiento a nuestra maravillosa gente ([emoji key](https://allcontributors.org/docs/en/emoji-key)): @@ -179,14 +260,18 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
Michael Shanks
📖 💻 ⚠️
Kevin Åberg Kultalahti
📖 💻 ⚠️
Joe
📖 💻 🖋 🎨 -
Conor_Mack
💻 ⚠️ +
Rory Powell
💻 📖 ⚠️ +
Peter Clement
💻 📖 ⚠️ +
Conor_Mack
💻 ⚠️
pngwn
💻 ⚠️
HugoLd
💻
victoriasloan
💻
yashank09
💻
SOVLOOKUP
💻 +
seoulaja
🌍 +
Maurits Lourens
⚠️ 💻 @@ -195,4 +280,5 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d -This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome! +Este proyecto sigue las especificaciones de [all-contributors](https://github.com/all-contributors/all-contributors). +Todo tipo de contribuciones son agradecidas! diff --git a/lerna.json b/lerna.json index e65ce2040..389d00264 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "1.0.212-alpha.6", + "version": "1.2.39-alpha.5", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/package.json b/package.json index 9c35af497..4c24e0025 100644 --- a/package.json +++ b/package.json @@ -25,8 +25,8 @@ "bootstrap": "lerna bootstrap && lerna link && ./scripts/link-dependencies.sh", "build": "lerna run build", "build:dev": "lerna run prebuild && tsc --build --watch --preserveWatchOutput", - "release": "lerna publish patch --yes --force-publish && yarn release:pro", - "release:develop": "lerna publish prerelease --yes --force-publish --dist-tag develop && yarn release:pro:develop", + "release": "lerna publish ${RELEASE_VERSION_TYPE:-patch} --yes --force-publish && yarn release:pro", + "release:develop": "lerna publish prerelease --yes --force-publish --dist-tag develop --exact && yarn release:pro:develop", "release:pro": "bash scripts/pro/release.sh", "release:pro:develop": "bash scripts/pro/release.sh develop", "restore": "yarn run clean && yarn run bootstrap && yarn run build", @@ -40,7 +40,8 @@ "dev": "yarn run kill-all && lerna link && lerna run --parallel dev:builder --concurrency 1", "dev:noserver": "yarn run kill-builder && lerna link && lerna run dev:stack:up && lerna run --parallel dev:builder --concurrency 1 --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker", "dev:server": "yarn run kill-server && lerna run --parallel dev:builder --concurrency 1 --scope @budibase/backend-core --scope @budibase/worker --scope @budibase/server", - "test": "lerna run test", + "test": "lerna run test && yarn test:pro", + "test:pro": "bash scripts/pro/test.sh", "lint:eslint": "eslint packages", "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\"", "lint": "yarn run lint:eslint && yarn run lint:prettier", @@ -53,6 +54,7 @@ "test:e2e:ci:notify": "lerna run cy:ci:notify", "build:specs": "lerna run specs", "build:docker": "lerna run build:docker && npm run build:docker:proxy:compose && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -", + "build:docker:pre": "lerna run build && lerna run predocker", "build:docker:proxy": "docker build hosting/proxy -t proxy-service", "build:docker:proxy:compose": "node scripts/proxy/generateProxyConfig compose && npm run build:docker:proxy", "build:docker:proxy:preprod": "node scripts/proxy/generateProxyConfig preprod && npm run build:docker:proxy", @@ -64,7 +66,7 @@ "build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -", "build:docker:single:multiarch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/single/Dockerfile -t budibase:latest .", "build:docker:single:image": "docker build -f hosting/single/Dockerfile -t budibase:latest .", - "build:docker:single": "lerna run build && lerna run predocker && npm run build:docker:single:image", + "build:docker:single": "npm run build:docker:pre && npm run build:docker:single:image", "build:docs": "lerna run build:docs", "release:helm": "node scripts/releaseHelmChart", "env:multi:enable": "lerna run env:multi:enable", @@ -83,4 +85,4 @@ "install:pro": "bash scripts/pro/install.sh", "dep:clean": "yarn clean && yarn bootstrap" } -} +} \ No newline at end of file diff --git a/packages/backend-core/cache.js b/packages/backend-core/cache.js index 6b319357c..c8bd3c9b6 100644 --- a/packages/backend-core/cache.js +++ b/packages/backend-core/cache.js @@ -5,4 +5,5 @@ module.exports = { app: require("./src/cache/appMetadata"), writethrough: require("./src/cache/writethrough"), ...generic, + cache: generic, } diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 3f3648212..93484c3b1 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "1.0.212-alpha.6", + "version": "1.2.39-alpha.5", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -20,13 +20,14 @@ "test:watch": "jest --watchAll" }, "dependencies": { - "@budibase/types": "^1.0.212-alpha.6", + "@budibase/types": "1.2.39-alpha.5", "@techpass/passport-openidconnect": "0.3.2", "aws-sdk": "2.1030.0", "bcrypt": "5.0.1", "dotenv": "16.0.1", "emitter-listener": "1.1.2", "ioredis": "4.28.0", + "joi": "17.6.0", "jsonwebtoken": "8.5.1", "koa-passport": "4.1.4", "lodash": "4.17.21", @@ -36,6 +37,7 @@ "passport-google-oauth": "2.0.0", "passport-jwt": "4.0.0", "passport-local": "1.0.0", + "passport-oauth2-refresh": "^2.1.0", "posthog-node": "1.3.0", "pouchdb": "7.3.0", "pouchdb-find": "7.2.2", @@ -61,6 +63,7 @@ "@shopify/jest-koa-mocks": "3.1.5", "@types/jest": "27.5.1", "@types/koa": "2.0.52", + "@types/lodash": "4.14.180", "@types/node": "14.18.20", "@types/node-fetch": "2.6.1", "@types/pouchdb": "6.4.0", diff --git a/packages/backend-core/src/auth.js b/packages/backend-core/src/auth.js index b13cd932c..d39b8426f 100644 --- a/packages/backend-core/src/auth.js +++ b/packages/backend-core/src/auth.js @@ -2,6 +2,9 @@ const passport = require("koa-passport") const LocalStrategy = require("passport-local").Strategy const JwtStrategy = require("passport-jwt").Strategy const { getGlobalDB } = require("./tenancy") +const refresh = require("passport-oauth2-refresh") +const { Configs } = require("./constants") +const { getScopedConfig } = require("./db/utils") const { jwt, local, @@ -12,10 +15,17 @@ const { tenancy, appTenancy, authError, + ssoCallbackUrl, csrf, internalApi, + adminOnly, + builderOnly, + builderOrAdmin, + joiValidator, } = require("./middleware") +const { invalidateUser } = require("./cache/user") + // Strategies passport.use(new LocalStrategy(local.options, local.authenticate)) passport.use(new JwtStrategy(jwt.options, jwt.authenticate)) @@ -34,6 +44,124 @@ passport.deserializeUser(async (user, done) => { } }) +async function refreshOIDCAccessToken(db, chosenConfig, refreshToken) { + const callbackUrl = await oidc.getCallbackUrl(db, chosenConfig) + let enrichedConfig + let strategy + + try { + enrichedConfig = await oidc.fetchStrategyConfig(chosenConfig, callbackUrl) + if (!enrichedConfig) { + throw new Error("OIDC Config contents invalid") + } + strategy = await oidc.strategyFactory(enrichedConfig) + } catch (err) { + console.error(err) + throw new Error("Could not refresh OAuth Token") + } + + refresh.use(strategy, { + setRefreshOAuth2() { + return strategy._getOAuth2Client(enrichedConfig) + }, + }) + + return new Promise(resolve => { + refresh.requestNewAccessToken( + Configs.OIDC, + refreshToken, + (err, accessToken, refreshToken, params) => { + resolve({ err, accessToken, refreshToken, params }) + } + ) + }) +} + +async function refreshGoogleAccessToken(db, config, refreshToken) { + let callbackUrl = await google.getCallbackUrl(db, config) + + let strategy + try { + strategy = await google.strategyFactory(config, callbackUrl) + } catch (err) { + console.error(err) + throw new Error("Error constructing OIDC refresh strategy", err) + } + + refresh.use(strategy) + + return new Promise(resolve => { + refresh.requestNewAccessToken( + Configs.GOOGLE, + refreshToken, + (err, accessToken, refreshToken, params) => { + resolve({ err, accessToken, refreshToken, params }) + } + ) + }) +} + +async function refreshOAuthToken(refreshToken, configType, configId) { + const db = getGlobalDB() + + const config = await getScopedConfig(db, { + type: configType, + group: {}, + }) + + let chosenConfig = {} + let refreshResponse + if (configType === Configs.OIDC) { + // configId - retrieved from cookie. + chosenConfig = config.configs.filter(c => c.uuid === configId)[0] + if (!chosenConfig) { + throw new Error("Invalid OIDC configuration") + } + refreshResponse = await refreshOIDCAccessToken( + db, + chosenConfig, + refreshToken + ) + } else { + chosenConfig = config + refreshResponse = await refreshGoogleAccessToken( + db, + chosenConfig, + refreshToken + ) + } + + return refreshResponse +} + +async function updateUserOAuth(userId, oAuthConfig) { + const details = { + accessToken: oAuthConfig.accessToken, + refreshToken: oAuthConfig.refreshToken, + } + + try { + const db = getGlobalDB() + const dbUser = await db.get(userId) + + //Do not overwrite the refresh token if a valid one is not provided. + if (typeof details.refreshToken !== "string") { + delete details.refreshToken + } + + dbUser.oauth2 = { + ...dbUser.oauth2, + ...details, + } + + await db.put(dbUser) + + await invalidateUser(userId) + } catch (e) { + console.error("Could not update OAuth details for current user", e) + } +} + module.exports = { buildAuthMiddleware: authenticated, passport, @@ -46,4 +174,11 @@ module.exports = { authError, buildCsrfMiddleware: csrf, internalApi, + refreshOAuthToken, + updateUserOAuth, + ssoCallbackUrl, + adminOnly, + builderOnly, + builderOrAdmin, + joiValidator, } diff --git a/packages/backend-core/src/cache/appMetadata.js b/packages/backend-core/src/cache/appMetadata.js index b0d9481cb..a7ff0d2fc 100644 --- a/packages/backend-core/src/cache/appMetadata.js +++ b/packages/backend-core/src/cache/appMetadata.js @@ -1,6 +1,6 @@ const redis = require("../redis/init") const { doWithDB } = require("../db") -const { DocumentTypes } = require("../db/constants") +const { DocumentType } = require("../db/constants") const AppState = { INVALID: "invalid", @@ -14,7 +14,7 @@ const populateFromDB = async appId => { return doWithDB( appId, db => { - return db.get(DocumentTypes.APP_METADATA) + return db.get(DocumentType.APP_METADATA) }, { skip_setup: true } ) diff --git a/packages/backend-core/src/cache/generic.js b/packages/backend-core/src/cache/generic.js index e2f391533..26ef0c6bb 100644 --- a/packages/backend-core/src/cache/generic.js +++ b/packages/backend-core/src/cache/generic.js @@ -9,6 +9,7 @@ exports.CacheKeys = { UNIQUE_TENANT_ID: "uniqueTenantId", EVENTS: "events", BACKFILL_METADATA: "backfillMetadata", + EVENTS_RATE_LIMIT: "eventsRateLimit", } exports.TTL = { diff --git a/packages/backend-core/src/cache/writethrough.ts b/packages/backend-core/src/cache/writethrough.ts index e11ca0aca..ec6b1604c 100644 --- a/packages/backend-core/src/cache/writethrough.ts +++ b/packages/backend-core/src/cache/writethrough.ts @@ -1,5 +1,6 @@ import BaseCache from "./base" import { getWritethroughClient } from "../redis/init" +import { logWarn } from "../logging" const DEFAULT_WRITE_RATE_MS = 10000 let CACHE: BaseCache | null = null @@ -51,10 +52,8 @@ export async function put( if (err.status !== 409) { throw err } else { - // get the rev, update over it - this is risky, may change in future - const readDoc = await db.get(doc._id) - doc._rev = readDoc._rev - await writeDb(doc) + // Swallow 409s but log them + logWarn(`Ignoring conflict in write-through cache`) } } } diff --git a/packages/backend-core/src/context/constants.ts b/packages/backend-core/src/context/constants.ts new file mode 100644 index 000000000..937ad8f24 --- /dev/null +++ b/packages/backend-core/src/context/constants.ts @@ -0,0 +1,17 @@ +export enum ContextKey { + TENANT_ID = "tenantId", + GLOBAL_DB = "globalDb", + APP_ID = "appId", + IDENTITY = "identity", + // whatever the request app DB was + CURRENT_DB = "currentDb", + // get the prod app DB from the request + PROD_DB = "prodDb", + // get the dev app DB from the request + DEV_DB = "devDb", + DB_OPTS = "dbOpts", + // check if something else is using the context, don't close DB + TENANCY_IN_USE = "tenancyInUse", + APP_IN_USE = "appInUse", + IDENTITY_IN_USE = "identityInUse", +} diff --git a/packages/backend-core/src/context/index.js b/packages/backend-core/src/context/index.js deleted file mode 100644 index a69a381f0..000000000 --- a/packages/backend-core/src/context/index.js +++ /dev/null @@ -1,353 +0,0 @@ -const env = require("../environment") -const { SEPARATOR, DocumentTypes } = require("../db/constants") -const { DEFAULT_TENANT_ID } = require("../constants") -const cls = require("./FunctionContext") -const { dangerousGetDB, closeDB } = require("../db") -const { getProdAppID, getDevelopmentAppID } = require("../db/conversions") -const { baseGlobalDBName } = require("../tenancy/utils") -const { isEqual } = require("lodash") - -// some test cases call functions directly, need to -// store an app ID to pretend there is a context -let TEST_APP_ID = null - -const ContextKeys = { - TENANT_ID: "tenantId", - GLOBAL_DB: "globalDb", - APP_ID: "appId", - IDENTITY: "identity", - // whatever the request app DB was - CURRENT_DB: "currentDb", - // get the prod app DB from the request - PROD_DB: "prodDb", - // get the dev app DB from the request - DEV_DB: "devDb", - DB_OPTS: "dbOpts", - // check if something else is using the context, don't close DB - IN_USE: "inUse", -} - -exports.DEFAULT_TENANT_ID = DEFAULT_TENANT_ID - -// this function makes sure the PouchDB objects are closed and -// fully deleted when finished - this protects against memory leaks -async function closeAppDBs() { - const dbKeys = [ - ContextKeys.CURRENT_DB, - ContextKeys.PROD_DB, - ContextKeys.DEV_DB, - ] - for (let dbKey of dbKeys) { - const db = cls.getFromContext(dbKey) - if (!db) { - continue - } - await closeDB(db) - // clear the DB from context, incase someone tries to use it again - cls.setOnContext(dbKey, null) - } - // clear the app ID now that the databases are closed - if (cls.getFromContext(ContextKeys.APP_ID)) { - cls.setOnContext(ContextKeys.APP_ID, null) - } - if (cls.getFromContext(ContextKeys.DB_OPTS)) { - cls.setOnContext(ContextKeys.DB_OPTS, null) - } -} - -exports.closeTenancy = async () => { - if (env.USE_COUCH) { - await closeDB(exports.getGlobalDB()) - } - // clear from context now that database is closed/task is finished - cls.setOnContext(ContextKeys.TENANT_ID, null) - cls.setOnContext(ContextKeys.GLOBAL_DB, null) -} - -exports.isDefaultTenant = () => { - return exports.getTenantId() === exports.DEFAULT_TENANT_ID -} - -exports.isMultiTenant = () => { - return env.MULTI_TENANCY -} - -// used for automations, API endpoints should always be in context already -exports.doInTenant = (tenantId, task, { forceNew } = {}) => { - // the internal function is so that we can re-use an existing - // context - don't want to close DB on a parent context - async function internal(opts = { existing: false }) { - // set the tenant id - if (!opts.existing) { - exports.updateTenantId(tenantId) - } - - try { - // invoke the task - return await task() - } finally { - const using = cls.getFromContext(ContextKeys.IN_USE) - if (!using || using <= 1) { - await exports.closeTenancy() - } else { - cls.setOnContext(using - 1) - } - } - } - - const using = cls.getFromContext(ContextKeys.IN_USE) - if ( - !forceNew && - using && - cls.getFromContext(ContextKeys.TENANT_ID) === tenantId - ) { - cls.setOnContext(ContextKeys.IN_USE, using + 1) - return internal({ existing: true }) - } else { - return cls.run(async () => { - cls.setOnContext(ContextKeys.IN_USE, 1) - return internal() - }) - } -} - -/** - * Given an app ID this will attempt to retrieve the tenant ID from it. - * @return {null|string} The tenant ID found within the app ID. - */ -exports.getTenantIDFromAppID = appId => { - if (!appId) { - return null - } - const split = appId.split(SEPARATOR) - const hasDev = split[1] === DocumentTypes.DEV - if ((hasDev && split.length === 3) || (!hasDev && split.length === 2)) { - return null - } - if (hasDev) { - return split[2] - } else { - return split[1] - } -} - -const setAppTenantId = appId => { - const appTenantId = - exports.getTenantIDFromAppID(appId) || exports.DEFAULT_TENANT_ID - exports.updateTenantId(appTenantId) -} - -exports.doInAppContext = (appId, task, { forceNew } = {}) => { - if (!appId) { - throw new Error("appId is required") - } - - const identity = exports.getIdentity() - - // the internal function is so that we can re-use an existing - // context - don't want to close DB on a parent context - async function internal(opts = { existing: false }) { - // set the app tenant id - if (!opts.existing) { - setAppTenantId(appId) - } - // set the app ID - cls.setOnContext(ContextKeys.APP_ID, appId) - // preserve the identity - exports.setIdentity(identity) - try { - // invoke the task - return await task() - } finally { - const using = cls.getFromContext(ContextKeys.IN_USE) - if (!using || using <= 1) { - await closeAppDBs() - } else { - cls.setOnContext(using - 1) - } - } - } - const using = cls.getFromContext(ContextKeys.IN_USE) - if (!forceNew && using && cls.getFromContext(ContextKeys.APP_ID) === appId) { - cls.setOnContext(ContextKeys.IN_USE, using + 1) - return internal({ existing: true }) - } else { - return cls.run(async () => { - cls.setOnContext(ContextKeys.IN_USE, 1) - return internal() - }) - } -} - -exports.doInIdentityContext = (identity, task) => { - if (!identity) { - throw new Error("identity is required") - } - - async function internal(opts = { existing: false }) { - if (!opts.existing) { - cls.setOnContext(ContextKeys.IDENTITY, identity) - // set the tenant so that doInTenant will preserve identity - if (identity.tenantId) { - exports.updateTenantId(identity.tenantId) - } - } - - try { - // invoke the task - return await task() - } finally { - const using = cls.getFromContext(ContextKeys.IN_USE) - if (!using || using <= 1) { - exports.setIdentity(null) - } else { - cls.setOnContext(using - 1) - } - } - } - - const existing = cls.getFromContext(ContextKeys.IDENTITY) - const using = cls.getFromContext(ContextKeys.IN_USE) - if (using && existing && existing._id === identity._id) { - cls.setOnContext(ContextKeys.IN_USE, using + 1) - return internal({ existing: true }) - } else { - return cls.run(async () => { - cls.setOnContext(ContextKeys.IN_USE, 1) - return internal({ existing: false }) - }) - } -} - -exports.setIdentity = identity => { - cls.setOnContext(ContextKeys.IDENTITY, identity) -} - -exports.getIdentity = () => { - try { - return cls.getFromContext(ContextKeys.IDENTITY) - } catch (e) { - // do nothing - identity is not in context - } -} - -exports.updateTenantId = tenantId => { - cls.setOnContext(ContextKeys.TENANT_ID, tenantId) - if (env.USE_COUCH) { - exports.setGlobalDB(tenantId) - } -} - -exports.updateAppId = async appId => { - try { - // have to close first, before removing the databases from context - await closeAppDBs() - cls.setOnContext(ContextKeys.APP_ID, appId) - } catch (err) { - if (env.isTest()) { - TEST_APP_ID = appId - } else { - throw err - } - } -} - -exports.setGlobalDB = tenantId => { - const dbName = baseGlobalDBName(tenantId) - const db = dangerousGetDB(dbName) - cls.setOnContext(ContextKeys.GLOBAL_DB, db) - return db -} - -exports.getGlobalDB = () => { - const db = cls.getFromContext(ContextKeys.GLOBAL_DB) - if (!db) { - throw new Error("Global DB not found") - } - return db -} - -exports.isTenantIdSet = () => { - const tenantId = cls.getFromContext(ContextKeys.TENANT_ID) - return !!tenantId -} - -exports.getTenantId = () => { - if (!exports.isMultiTenant()) { - return exports.DEFAULT_TENANT_ID - } - const tenantId = cls.getFromContext(ContextKeys.TENANT_ID) - if (!tenantId) { - throw new Error("Tenant id not found") - } - return tenantId -} - -exports.getAppId = () => { - const foundId = cls.getFromContext(ContextKeys.APP_ID) - if (!foundId && env.isTest() && TEST_APP_ID) { - return TEST_APP_ID - } else { - return foundId - } -} - -function getContextDB(key, opts) { - const dbOptsKey = `${key}${ContextKeys.DB_OPTS}` - let storedOpts = cls.getFromContext(dbOptsKey) - let db = cls.getFromContext(key) - if (db && isEqual(opts, storedOpts)) { - return db - } - - const appId = exports.getAppId() - let toUseAppId - - switch (key) { - case ContextKeys.CURRENT_DB: - toUseAppId = appId - break - case ContextKeys.PROD_DB: - toUseAppId = getProdAppID(appId) - break - case ContextKeys.DEV_DB: - toUseAppId = getDevelopmentAppID(appId) - break - } - db = dangerousGetDB(toUseAppId, opts) - try { - cls.setOnContext(key, db) - if (opts) { - cls.setOnContext(dbOptsKey, opts) - } - } catch (err) { - if (!env.isTest()) { - throw err - } - } - return db -} - -/** - * Opens the app database based on whatever the request - * contained, dev or prod. - */ -exports.getAppDB = (opts = null) => { - return getContextDB(ContextKeys.CURRENT_DB, opts) -} - -/** - * This specifically gets the prod app ID, if the request - * contained a development app ID, this will open the prod one. - */ -exports.getProdAppDB = (opts = null) => { - return getContextDB(ContextKeys.PROD_DB, opts) -} - -/** - * This specifically gets the dev app ID, if the request - * contained a prod app ID, this will open the dev one. - */ -exports.getDevAppDB = (opts = null) => { - return getContextDB(ContextKeys.DEV_DB, opts) -} diff --git a/packages/backend-core/src/context/index.ts b/packages/backend-core/src/context/index.ts new file mode 100644 index 000000000..78ce764d5 --- /dev/null +++ b/packages/backend-core/src/context/index.ts @@ -0,0 +1,251 @@ +import env from "../environment" +import { SEPARATOR, DocumentType } from "../db/constants" +import cls from "./FunctionContext" +import { dangerousGetDB, closeDB } from "../db" +import { baseGlobalDBName } from "../tenancy/utils" +import { IdentityContext } from "@budibase/types" +import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants" +import { ContextKey } from "./constants" +import { + updateUsing, + closeWithUsing, + setAppTenantId, + setIdentity, + closeAppDBs, + getContextDB, +} from "./utils" + +export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID + +// some test cases call functions directly, need to +// store an app ID to pretend there is a context +let TEST_APP_ID: string | null = null + +export const closeTenancy = async () => { + let db + try { + if (env.USE_COUCH) { + db = getGlobalDB() + } + } catch (err) { + // no DB found - skip closing + return + } + await closeDB(db) + // clear from context now that database is closed/task is finished + cls.setOnContext(ContextKey.TENANT_ID, null) + cls.setOnContext(ContextKey.GLOBAL_DB, null) +} + +// export const isDefaultTenant = () => { +// return getTenantId() === DEFAULT_TENANT_ID +// } + +export const isMultiTenant = () => { + return env.MULTI_TENANCY +} + +/** + * Given an app ID this will attempt to retrieve the tenant ID from it. + * @return {null|string} The tenant ID found within the app ID. + */ +export const getTenantIDFromAppID = (appId: string) => { + if (!appId) { + return null + } + const split = appId.split(SEPARATOR) + const hasDev = split[1] === DocumentType.DEV + if ((hasDev && split.length === 3) || (!hasDev && split.length === 2)) { + return null + } + if (hasDev) { + return split[2] + } else { + return split[1] + } +} + +// used for automations, API endpoints should always be in context already +export const doInTenant = (tenantId: string | null, task: any) => { + // make sure default always selected in single tenancy + if (!env.MULTI_TENANCY) { + tenantId = tenantId || DEFAULT_TENANT_ID + } + // the internal function is so that we can re-use an existing + // context - don't want to close DB on a parent context + async function internal(opts = { existing: false }) { + // set the tenant id + global db if this is a new context + if (!opts.existing) { + updateTenantId(tenantId) + } + + try { + // invoke the task + return await task() + } finally { + await closeWithUsing(ContextKey.TENANCY_IN_USE, () => { + return closeTenancy() + }) + } + } + + const existing = cls.getFromContext(ContextKey.TENANT_ID) === tenantId + return updateUsing(ContextKey.TENANCY_IN_USE, existing, internal) +} + +export const doInAppContext = (appId: string, task: any) => { + if (!appId) { + throw new Error("appId is required") + } + + const identity = getIdentity() + + // the internal function is so that we can re-use an existing + // context - don't want to close DB on a parent context + async function internal(opts = { existing: false }) { + // set the app tenant id + if (!opts.existing) { + setAppTenantId(appId) + } + // set the app ID + cls.setOnContext(ContextKey.APP_ID, appId) + + // preserve the identity + if (identity) { + setIdentity(identity) + } + try { + // invoke the task + return await task() + } finally { + await closeWithUsing(ContextKey.APP_IN_USE, async () => { + await closeAppDBs() + await closeTenancy() + }) + } + } + const existing = cls.getFromContext(ContextKey.APP_ID) === appId + return updateUsing(ContextKey.APP_IN_USE, existing, internal) +} + +export const doInIdentityContext = (identity: IdentityContext, task: any) => { + if (!identity) { + throw new Error("identity is required") + } + + async function internal(opts = { existing: false }) { + if (!opts.existing) { + cls.setOnContext(ContextKey.IDENTITY, identity) + // set the tenant so that doInTenant will preserve identity + if (identity.tenantId) { + updateTenantId(identity.tenantId) + } + } + + try { + // invoke the task + return await task() + } finally { + await closeWithUsing(ContextKey.IDENTITY_IN_USE, async () => { + setIdentity(null) + await closeTenancy() + }) + } + } + + const existing = cls.getFromContext(ContextKey.IDENTITY) + return updateUsing(ContextKey.IDENTITY_IN_USE, existing, internal) +} + +export const getIdentity = (): IdentityContext | undefined => { + try { + return cls.getFromContext(ContextKey.IDENTITY) + } catch (e) { + // do nothing - identity is not in context + } +} + +export const updateTenantId = (tenantId: string | null) => { + cls.setOnContext(ContextKey.TENANT_ID, tenantId) + if (env.USE_COUCH) { + setGlobalDB(tenantId) + } +} + +export const updateAppId = async (appId: string) => { + try { + // have to close first, before removing the databases from context + await closeAppDBs() + cls.setOnContext(ContextKey.APP_ID, appId) + } catch (err) { + if (env.isTest()) { + TEST_APP_ID = appId + } else { + throw err + } + } +} + +export const setGlobalDB = (tenantId: string | null) => { + const dbName = baseGlobalDBName(tenantId) + const db = dangerousGetDB(dbName) + cls.setOnContext(ContextKey.GLOBAL_DB, db) + return db +} + +export const getGlobalDB = () => { + const db = cls.getFromContext(ContextKey.GLOBAL_DB) + if (!db) { + throw new Error("Global DB not found") + } + return db +} + +export const isTenantIdSet = () => { + const tenantId = cls.getFromContext(ContextKey.TENANT_ID) + return !!tenantId +} + +export const getTenantId = () => { + if (!isMultiTenant()) { + return DEFAULT_TENANT_ID + } + const tenantId = cls.getFromContext(ContextKey.TENANT_ID) + if (!tenantId) { + throw new Error("Tenant id not found") + } + return tenantId +} + +export const getAppId = () => { + const foundId = cls.getFromContext(ContextKey.APP_ID) + if (!foundId && env.isTest() && TEST_APP_ID) { + return TEST_APP_ID + } else { + return foundId + } +} + +/** + * Opens the app database based on whatever the request + * contained, dev or prod. + */ +export const getAppDB = (opts?: any) => { + return getContextDB(ContextKey.CURRENT_DB, opts) +} + +/** + * This specifically gets the prod app ID, if the request + * contained a development app ID, this will open the prod one. + */ +export const getProdAppDB = (opts?: any) => { + return getContextDB(ContextKey.PROD_DB, opts) +} + +/** + * This specifically gets the dev app ID, if the request + * contained a prod app ID, this will open the dev one. + */ +export const getDevAppDB = (opts?: any) => { + return getContextDB(ContextKey.DEV_DB, opts) +} diff --git a/packages/backend-core/src/context/tests/index.spec.ts b/packages/backend-core/src/context/tests/index.spec.ts new file mode 100644 index 000000000..55ecd333a --- /dev/null +++ b/packages/backend-core/src/context/tests/index.spec.ts @@ -0,0 +1,148 @@ +import "../../../tests/utilities/TestConfiguration" +import * as context from ".." +import { DEFAULT_TENANT_ID } from "../../constants" +import env from "../../environment" + +// must use require to spy index file exports due to known issue in jest +const dbUtils = require("../../db") +jest.spyOn(dbUtils, "closeDB") +jest.spyOn(dbUtils, "dangerousGetDB") + +describe("context", () => { + beforeEach(() => { + jest.clearAllMocks() + }) + + describe("doInTenant", () => { + describe("single-tenancy", () => { + it("defaults to the default tenant", () => { + const tenantId = context.getTenantId() + expect(tenantId).toBe(DEFAULT_TENANT_ID) + }) + + it("defaults to the default tenant db", async () => { + await context.doInTenant(DEFAULT_TENANT_ID, () => { + const db = context.getGlobalDB() + expect(db.name).toBe("global-db") + }) + expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1) + expect(dbUtils.closeDB).toHaveBeenCalledTimes(1) + }) + }) + + describe("multi-tenancy", () => { + beforeEach(() => { + env._set("MULTI_TENANCY", 1) + }) + + it("fails when no tenant id is set", () => { + const test = () => { + let error + try { + context.getTenantId() + } catch (e: any) { + error = e + } + expect(error.message).toBe("Tenant id not found") + } + + // test under no tenancy + test() + + // test after tenancy has been accessed to ensure cleanup + context.doInTenant("test", () => {}) + test() + }) + + it("fails when no tenant db is set", () => { + const test = () => { + let error + try { + context.getGlobalDB() + } catch (e: any) { + error = e + } + expect(error.message).toBe("Global DB not found") + } + + // test under no tenancy + test() + + // test after tenancy has been accessed to ensure cleanup + context.doInTenant("test", () => {}) + test() + }) + + it("sets tenant id", () => { + context.doInTenant("test", () => { + const tenantId = context.getTenantId() + expect(tenantId).toBe("test") + }) + }) + + it("initialises the tenant db", async () => { + await context.doInTenant("test", () => { + const db = context.getGlobalDB() + expect(db.name).toBe("test_global-db") + }) + expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1) + expect(dbUtils.closeDB).toHaveBeenCalledTimes(1) + }) + + it("sets the tenant id when nested with same tenant id", async () => { + await context.doInTenant("test", async () => { + const tenantId = context.getTenantId() + expect(tenantId).toBe("test") + + await context.doInTenant("test", async () => { + const tenantId = context.getTenantId() + expect(tenantId).toBe("test") + + await context.doInTenant("test", () => { + const tenantId = context.getTenantId() + expect(tenantId).toBe("test") + }) + }) + }) + }) + + it("initialises the tenant db when nested with same tenant id", async () => { + await context.doInTenant("test", async () => { + const db = context.getGlobalDB() + expect(db.name).toBe("test_global-db") + + await context.doInTenant("test", async () => { + const db = context.getGlobalDB() + expect(db.name).toBe("test_global-db") + + await context.doInTenant("test", () => { + const db = context.getGlobalDB() + expect(db.name).toBe("test_global-db") + }) + }) + }) + + // only 1 db is opened and closed + expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1) + expect(dbUtils.closeDB).toHaveBeenCalledTimes(1) + }) + + it("sets different tenant id inside another context", () => { + context.doInTenant("test", () => { + const tenantId = context.getTenantId() + expect(tenantId).toBe("test") + + context.doInTenant("nested", () => { + const tenantId = context.getTenantId() + expect(tenantId).toBe("nested") + + context.doInTenant("double-nested", () => { + const tenantId = context.getTenantId() + expect(tenantId).toBe("double-nested") + }) + }) + }) + }) + }) + }) +}) diff --git a/packages/backend-core/src/context/utils.ts b/packages/backend-core/src/context/utils.ts new file mode 100644 index 000000000..6e7100b59 --- /dev/null +++ b/packages/backend-core/src/context/utils.ts @@ -0,0 +1,109 @@ +import { + DEFAULT_TENANT_ID, + getAppId, + getTenantIDFromAppID, + updateTenantId, +} from "./index" +import cls from "./FunctionContext" +import { IdentityContext } from "@budibase/types" +import { ContextKey } from "./constants" +import { dangerousGetDB, closeDB } from "../db" +import { isEqual } from "lodash" +import { getDevelopmentAppID, getProdAppID } from "../db/conversions" +import env from "../environment" + +export async function updateUsing( + usingKey: string, + existing: boolean, + internal: (opts: { existing: boolean }) => Promise +) { + const using = cls.getFromContext(usingKey) + if (using && existing) { + cls.setOnContext(usingKey, using + 1) + return internal({ existing: true }) + } else { + return cls.run(async () => { + cls.setOnContext(usingKey, 1) + return internal({ existing: false }) + }) + } +} + +export async function closeWithUsing( + usingKey: string, + closeFn: () => Promise +) { + const using = cls.getFromContext(usingKey) + if (!using || using <= 1) { + await closeFn() + } else { + cls.setOnContext(usingKey, using - 1) + } +} + +export const setAppTenantId = (appId: string) => { + const appTenantId = getTenantIDFromAppID(appId) || DEFAULT_TENANT_ID + updateTenantId(appTenantId) +} + +export const setIdentity = (identity: IdentityContext | null) => { + cls.setOnContext(ContextKey.IDENTITY, identity) +} + +// this function makes sure the PouchDB objects are closed and +// fully deleted when finished - this protects against memory leaks +export async function closeAppDBs() { + const dbKeys = [ContextKey.CURRENT_DB, ContextKey.PROD_DB, ContextKey.DEV_DB] + for (let dbKey of dbKeys) { + const db = cls.getFromContext(dbKey) + if (!db) { + continue + } + await closeDB(db) + // clear the DB from context, incase someone tries to use it again + cls.setOnContext(dbKey, null) + } + // clear the app ID now that the databases are closed + if (cls.getFromContext(ContextKey.APP_ID)) { + cls.setOnContext(ContextKey.APP_ID, null) + } + if (cls.getFromContext(ContextKey.DB_OPTS)) { + cls.setOnContext(ContextKey.DB_OPTS, null) + } +} + +export function getContextDB(key: string, opts: any) { + const dbOptsKey = `${key}${ContextKey.DB_OPTS}` + let storedOpts = cls.getFromContext(dbOptsKey) + let db = cls.getFromContext(key) + if (db && isEqual(opts, storedOpts)) { + return db + } + + const appId = getAppId() + let toUseAppId + + switch (key) { + case ContextKey.CURRENT_DB: + toUseAppId = appId + break + case ContextKey.PROD_DB: + toUseAppId = getProdAppID(appId) + break + case ContextKey.DEV_DB: + toUseAppId = getDevelopmentAppID(appId) + break + } + db = dangerousGetDB(toUseAppId, opts) + try { + cls.setOnContext(key, db) + if (opts) { + cls.setOnContext(dbOptsKey, opts) + } + } catch (err) { + if (!env.isTest()) { + throw err + } + } + return db +} diff --git a/packages/backend-core/src/db/constants.js b/packages/backend-core/src/db/constants.js deleted file mode 100644 index 10c6e174d..000000000 --- a/packages/backend-core/src/db/constants.js +++ /dev/null @@ -1,41 +0,0 @@ -exports.SEPARATOR = "_" - -const PRE_APP = "app" -const PRE_DEV = "dev" - -exports.DocumentTypes = { - USER: "us", - WORKSPACE: "workspace", - CONFIG: "config", - TEMPLATE: "template", - APP: PRE_APP, - DEV: PRE_DEV, - APP_DEV: `${PRE_APP}${exports.SEPARATOR}${PRE_DEV}`, - APP_METADATA: `${PRE_APP}${exports.SEPARATOR}metadata`, - ROLE: "role", - MIGRATIONS: "migrations", - DEV_INFO: "devinfo", -} - -exports.StaticDatabases = { - GLOBAL: { - name: "global-db", - docs: { - apiKeys: "apikeys", - usageQuota: "usage_quota", - licenseInfo: "license_info", - }, - }, - // contains information about tenancy and so on - PLATFORM_INFO: { - name: "global-info", - docs: { - tenants: "tenants", - install: "install", - }, - }, -} - -exports.APP_PREFIX = exports.DocumentTypes.APP + exports.SEPARATOR -exports.APP_DEV = exports.APP_DEV_PREFIX = - exports.DocumentTypes.APP_DEV + exports.SEPARATOR diff --git a/packages/backend-core/src/db/constants.ts b/packages/backend-core/src/db/constants.ts new file mode 100644 index 000000000..460476da2 --- /dev/null +++ b/packages/backend-core/src/db/constants.ts @@ -0,0 +1,67 @@ +export const SEPARATOR = "_" +export const UNICODE_MAX = "\ufff0" + +/** + * Can be used to create a few different forms of querying a view. + */ +export enum AutomationViewMode { + ALL = "all", + AUTOMATION = "automation", + STATUS = "status", +} + +export enum ViewName { + USER_BY_APP = "by_app", + USER_BY_EMAIL = "by_email2", + BY_API_KEY = "by_api_key", + USER_BY_BUILDERS = "by_builders", + LINK = "by_link", + ROUTING = "screen_routes", + AUTOMATION_LOGS = "automation_logs", +} + +export const DeprecatedViews = { + [ViewName.USER_BY_EMAIL]: [ + // removed due to inaccuracy in view doc filter logic + "by_email", + ], +} + +export enum DocumentType { + USER = "us", + GROUP = "gr", + WORKSPACE = "workspace", + CONFIG = "config", + TEMPLATE = "template", + APP = "app", + DEV = "dev", + APP_DEV = "app_dev", + APP_METADATA = "app_metadata", + ROLE = "role", + MIGRATIONS = "migrations", + DEV_INFO = "devinfo", + AUTOMATION_LOG = "log_au", +} + +export const StaticDatabases = { + GLOBAL: { + name: "global-db", + docs: { + apiKeys: "apikeys", + usageQuota: "usage_quota", + licenseInfo: "license_info", + }, + }, + // contains information about tenancy and so on + PLATFORM_INFO: { + name: "global-info", + docs: { + tenants: "tenants", + install: "install", + }, + }, +} + +export const APP_PREFIX = DocumentType.APP + SEPARATOR +export const APP_DEV = DocumentType.APP_DEV + SEPARATOR +export const APP_DEV_PREFIX = APP_DEV diff --git a/packages/backend-core/src/db/conversions.js b/packages/backend-core/src/db/conversions.js index 455cc712d..90c04e925 100644 --- a/packages/backend-core/src/db/conversions.js +++ b/packages/backend-core/src/db/conversions.js @@ -50,3 +50,8 @@ exports.getProdAppID = appId => { const rest = split.join(APP_DEV_PREFIX) return `${APP_PREFIX}${rest}` } + +exports.extractAppUUID = id => { + const split = id?.split("_") || [] + return split.length ? split[split.length - 1] : null +} diff --git a/packages/backend-core/src/db/index.js b/packages/backend-core/src/db/index.js index 8124be979..aa6f7ebc2 100644 --- a/packages/backend-core/src/db/index.js +++ b/packages/backend-core/src/db/index.js @@ -1,10 +1,18 @@ const pouch = require("./pouch") const env = require("../environment") +const openDbs = [] let PouchDB let initialised = false const dbList = new Set() +if (env.MEMORY_LEAK_CHECK) { + setInterval(() => { + console.log("--- OPEN DBS ---") + console.log(openDbs) + }, 5000) +} + const put = dbPut => async (doc, options = {}) => { @@ -35,6 +43,9 @@ exports.dangerousGetDB = (dbName, opts) => { dbList.add(dbName) } const db = new PouchDB(dbName, opts) + if (env.MEMORY_LEAK_CHECK) { + openDbs.push(db.name) + } const dbPut = db.put db.put = put(dbPut) return db @@ -46,6 +57,9 @@ exports.closeDB = async db => { if (!db || env.isTest()) { return } + if (env.MEMORY_LEAK_CHECK) { + openDbs.splice(openDbs.indexOf(db.name), 1) + } try { // specifically await so that if there is an error, it can be ignored return await db.close() diff --git a/packages/backend-core/src/db/pouch.js b/packages/backend-core/src/db/pouch.js index 59b7ff8ae..12d7d787e 100644 --- a/packages/backend-core/src/db/pouch.js +++ b/packages/backend-core/src/db/pouch.js @@ -102,6 +102,13 @@ exports.getPouch = (opts = {}) => { } } + if (opts.onDisk) { + POUCH_DB_DEFAULTS = { + prefix: undefined, + adapter: "leveldb", + } + } + if (opts.replication) { const replicationStream = require("pouchdb-replication-stream") PouchDB.plugin(replicationStream.plugin) diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index dc7a0454c..321ebd7f5 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -1,25 +1,17 @@ import { newid } from "../hashing" import { DEFAULT_TENANT_ID, Configs } from "../constants" import env from "../environment" -import { SEPARATOR, DocumentTypes } from "./constants" +import { SEPARATOR, DocumentType, UNICODE_MAX, ViewName } from "./constants" import { getTenantId, getGlobalDBName, getGlobalDB } from "../tenancy" import fetch from "node-fetch" import { doWithDB, allDbs } from "./index" import { getCouchInfo } from "./pouch" import { getAppMetadata } from "../cache/appMetadata" import { checkSlashesInUrl } from "../helpers" -import { isDevApp, isDevAppID } from "./conversions" +import { isDevApp, isDevAppID, getProdAppID } from "./conversions" import { APP_PREFIX } from "./constants" import * as events from "../events" -const UNICODE_MAX = "\ufff0" - -export const ViewNames = { - USER_BY_EMAIL: "by_email", - BY_API_KEY: "by_api_key", - USER_BY_BUILDERS: "by_builders", -} - export * from "./constants" export * from "./conversions" export { default as Replication } from "./Replication" @@ -63,12 +55,19 @@ export function getDocParams( } } +/** + * Retrieve the correct index for a view based on default design DB. + */ +export function getQueryIndex(viewName: ViewName) { + return `database/${viewName}` +} + /** * Generates a new workspace ID. * @returns {string} The new workspace ID which the workspace doc can be stored under. */ export function generateWorkspaceID() { - return `${DocumentTypes.WORKSPACE}${SEPARATOR}${newid()}` + return `${DocumentType.WORKSPACE}${SEPARATOR}${newid()}` } /** @@ -77,8 +76,8 @@ export function generateWorkspaceID() { export function getWorkspaceParams(id = "", otherProps = {}) { return { ...otherProps, - startkey: `${DocumentTypes.WORKSPACE}${SEPARATOR}${id}`, - endkey: `${DocumentTypes.WORKSPACE}${SEPARATOR}${id}${UNICODE_MAX}`, + startkey: `${DocumentType.WORKSPACE}${SEPARATOR}${id}`, + endkey: `${DocumentType.WORKSPACE}${SEPARATOR}${id}${UNICODE_MAX}`, } } @@ -87,20 +86,33 @@ export function getWorkspaceParams(id = "", otherProps = {}) { * @returns {string} The new user ID which the user doc can be stored under. */ export function generateGlobalUserID(id?: any) { - return `${DocumentTypes.USER}${SEPARATOR}${id || newid()}` + return `${DocumentType.USER}${SEPARATOR}${id || newid()}` } /** * Gets parameters for retrieving users. */ -export function getGlobalUserParams(globalId: any, otherProps = {}) { +export function getGlobalUserParams(globalId: any, otherProps: any = {}) { if (!globalId) { globalId = "" } + const startkey = otherProps?.startkey return { ...otherProps, - startkey: `${DocumentTypes.USER}${SEPARATOR}${globalId}`, - endkey: `${DocumentTypes.USER}${SEPARATOR}${globalId}${UNICODE_MAX}`, + // need to include this incase pagination + startkey: startkey + ? startkey + : `${DocumentType.USER}${SEPARATOR}${globalId}`, + endkey: `${DocumentType.USER}${SEPARATOR}${globalId}${UNICODE_MAX}`, + } +} + +export function getUsersByAppParams(appId: any, otherProps: any = {}) { + const prodAppId = getProdAppID(appId) + return { + ...otherProps, + startkey: prodAppId, + endkey: `${prodAppId}${UNICODE_MAX}`, } } @@ -109,7 +121,11 @@ export function getGlobalUserParams(globalId: any, otherProps = {}) { * @param ownerId The owner/user of the template, this could be global or a workspace level. */ export function generateTemplateID(ownerId: any) { - return `${DocumentTypes.TEMPLATE}${SEPARATOR}${ownerId}${SEPARATOR}${newid()}` + return `${DocumentType.TEMPLATE}${SEPARATOR}${ownerId}${SEPARATOR}${newid()}` +} + +export function generateAppUserID(prodAppId: string, userId: string) { + return `${prodAppId}${SEPARATOR}${userId}` } /** @@ -127,7 +143,7 @@ export function getTemplateParams( if (templateId) { final = templateId } else { - final = `${DocumentTypes.TEMPLATE}${SEPARATOR}${ownerId}${SEPARATOR}` + final = `${DocumentType.TEMPLATE}${SEPARATOR}${ownerId}${SEPARATOR}` } return { ...otherProps, @@ -141,14 +157,14 @@ export function getTemplateParams( * @returns {string} The new role ID which the role doc can be stored under. */ export function generateRoleID(id: any) { - return `${DocumentTypes.ROLE}${SEPARATOR}${id || newid()}` + return `${DocumentType.ROLE}${SEPARATOR}${id || newid()}` } /** * Gets parameters for retrieving a role, this is a utility function for the getDocParams function. */ export function getRoleParams(roleId = null, otherProps = {}) { - return getDocParams(DocumentTypes.ROLE, roleId, otherProps) + return getDocParams(DocumentType.ROLE, roleId, otherProps) } export function getStartEndKeyURL(base: any, baseKey: any, tenantId = null) { @@ -195,9 +211,9 @@ export async function getAllDbs(opts = { efficient: false }) { await addDbs(couchUrl) } else { // get prod apps - await addDbs(getStartEndKeyURL(couchUrl, DocumentTypes.APP, tenantId)) + await addDbs(getStartEndKeyURL(couchUrl, DocumentType.APP, tenantId)) // get dev apps - await addDbs(getStartEndKeyURL(couchUrl, DocumentTypes.APP_DEV, tenantId)) + await addDbs(getStartEndKeyURL(couchUrl, DocumentType.APP_DEV, tenantId)) // add global db name dbs.push(getGlobalDBName(tenantId)) } @@ -217,14 +233,18 @@ export async function getAllApps({ dev, all, idsOnly, efficient }: any = {}) { } let dbs = await getAllDbs({ efficient }) const appDbNames = dbs.filter((dbName: any) => { + if (env.isTest() && !dbName) { + return false + } + const split = dbName.split(SEPARATOR) // it is an app, check the tenantId - if (split[0] === DocumentTypes.APP) { + if (split[0] === DocumentType.APP) { // tenantId is always right before the UUID const possibleTenantId = split[split.length - 2] const noTenantId = - split.length === 2 || possibleTenantId === DocumentTypes.DEV + split.length === 2 || possibleTenantId === DocumentType.DEV return ( (tenantId === DEFAULT_TENANT_ID && noTenantId) || @@ -310,7 +330,7 @@ export async function dbExists(dbName: any) { export const generateConfigID = ({ type, workspace, user }: any) => { const scope = [type, workspace, user].filter(Boolean).join(SEPARATOR) - return `${DocumentTypes.CONFIG}${SEPARATOR}${scope}` + return `${DocumentType.CONFIG}${SEPARATOR}${scope}` } /** @@ -324,8 +344,8 @@ export const getConfigParams = ( return { ...otherProps, - startkey: `${DocumentTypes.CONFIG}${SEPARATOR}${scope}`, - endkey: `${DocumentTypes.CONFIG}${SEPARATOR}${scope}${UNICODE_MAX}`, + startkey: `${DocumentType.CONFIG}${SEPARATOR}${scope}`, + endkey: `${DocumentType.CONFIG}${SEPARATOR}${scope}${UNICODE_MAX}`, } } @@ -334,7 +354,7 @@ export const getConfigParams = ( * @returns {string} The new dev info ID which info for dev (like api key) can be stored under. */ export const generateDevInfoID = (userId: any) => { - return `${DocumentTypes.DEV_INFO}${SEPARATOR}${userId}` + return `${DocumentType.DEV_INFO}${SEPARATOR}${userId}` } /** @@ -384,7 +404,9 @@ export const getScopedFullConfig = async function ( if (type === Configs.SETTINGS) { if (scopedConfig && scopedConfig.doc) { // overrides affected by environment variables - scopedConfig.doc.config.platformUrl = await getPlatformUrl() + scopedConfig.doc.config.platformUrl = await getPlatformUrl({ + tenantAware: true, + }) scopedConfig.doc.config.analyticsEnabled = await events.analytics.enabled() } else { @@ -393,7 +415,7 @@ export const getScopedFullConfig = async function ( doc: { _id: generateConfigID({ type, user, workspace }), config: { - platformUrl: await getPlatformUrl(), + platformUrl: await getPlatformUrl({ tenantAware: true }), analyticsEnabled: await events.analytics.enabled(), }, }, @@ -434,6 +456,40 @@ export const getPlatformUrl = async (opts = { tenantAware: true }) => { return platformUrl } +export function pagination( + data: any[], + pageSize: number, + { + paginate, + property, + getKey, + }: { + paginate: boolean + property: string + getKey?: (doc: any) => string | undefined + } = { + paginate: true, + property: "_id", + } +) { + if (!paginate) { + return { data, hasNextPage: false } + } + const hasNextPage = data.length > pageSize + let nextPage = undefined + if (!getKey) { + getKey = (doc: any) => (property ? doc?.[property] : doc?._id) + } + if (hasNextPage) { + nextPage = getKey(data[pageSize]) + } + return { + data: data.slice(0, pageSize), + hasNextPage, + nextPage, + } +} + export async function getScopedConfig(db: any, params: any) { const configDoc = await getScopedFullConfig(db, params) return configDoc && configDoc.config ? configDoc.config : configDoc diff --git a/packages/backend-core/src/db/views.js b/packages/backend-core/src/db/views.js index e0281c658..3a45611a8 100644 --- a/packages/backend-core/src/db/views.js +++ b/packages/backend-core/src/db/views.js @@ -1,20 +1,42 @@ -const { DocumentTypes, ViewNames } = require("./utils") +const { + DocumentType, + ViewName, + DeprecatedViews, + SEPARATOR, +} = require("./utils") const { getGlobalDB } = require("../tenancy") +const DESIGN_DB = "_design/database" + function DesignDoc() { return { - _id: "_design/database", + _id: DESIGN_DB, // view collation information, read before writing any complex views: // https://docs.couchdb.org/en/master/ddocs/views/collation.html#collation-specification views: {}, } } -exports.createUserEmailView = async () => { +async function removeDeprecated(db, viewName) { + if (!DeprecatedViews[viewName]) { + return + } + try { + const designDoc = await db.get(DESIGN_DB) + for (let deprecatedNames of DeprecatedViews[viewName]) { + delete designDoc.views[deprecatedNames] + } + await db.put(designDoc) + } catch (err) { + // doesn't exist, ignore + } +} + +exports.createNewUserEmailView = async () => { const db = getGlobalDB() let designDoc try { - designDoc = await db.get("_design/database") + designDoc = await db.get(DESIGN_DB) } catch (err) { // no design doc, make one designDoc = DesignDoc() @@ -22,14 +44,41 @@ exports.createUserEmailView = async () => { const view = { // if using variables in a map function need to inject them before use map: `function(doc) { - if (doc._id.startsWith("${DocumentTypes.USER}")) { + if (doc._id.startsWith("${DocumentType.USER}${SEPARATOR}")) { emit(doc.email.toLowerCase(), doc._id) } }`, } designDoc.views = { ...designDoc.views, - [ViewNames.USER_BY_EMAIL]: view, + [ViewName.USER_BY_EMAIL]: view, + } + await db.put(designDoc) +} + +exports.createUserAppView = async () => { + const db = getGlobalDB() + let designDoc + try { + designDoc = await db.get("_design/database") + } catch (err) { + // no design doc, make one + designDoc = DesignDoc() + } + const view = { + // if using variables in a map function need to inject them before use + map: `function(doc) { + if (doc._id.startsWith("${DocumentType.USER}${SEPARATOR}") && doc.roles) { + for (let prodAppId of Object.keys(doc.roles)) { + let emitted = prodAppId + "${SEPARATOR}" + doc._id + emit(emitted, null) + } + } + }`, + } + designDoc.views = { + ...designDoc.views, + [ViewName.USER_BY_APP]: view, } await db.put(designDoc) } @@ -44,14 +93,14 @@ exports.createApiKeyView = async () => { } const view = { map: `function(doc) { - if (doc._id.startsWith("${DocumentTypes.DEV_INFO}") && doc.apiKey) { + if (doc._id.startsWith("${DocumentType.DEV_INFO}") && doc.apiKey) { emit(doc.apiKey, doc.userId) } }`, } designDoc.views = { ...designDoc.views, - [ViewNames.BY_API_KEY]: view, + [ViewName.BY_API_KEY]: view, } await db.put(designDoc) } @@ -74,16 +123,17 @@ exports.createUserBuildersView = async () => { } designDoc.views = { ...designDoc.views, - [ViewNames.USER_BY_BUILDERS]: view, + [ViewName.USER_BY_BUILDERS]: view, } await db.put(designDoc) } exports.queryGlobalView = async (viewName, params, db = null) => { const CreateFuncByName = { - [ViewNames.USER_BY_EMAIL]: exports.createUserEmailView, - [ViewNames.BY_API_KEY]: exports.createApiKeyView, - [ViewNames.USER_BY_BUILDERS]: exports.createUserBuildersView, + [ViewName.USER_BY_EMAIL]: exports.createNewUserEmailView, + [ViewName.BY_API_KEY]: exports.createApiKeyView, + [ViewName.USER_BY_BUILDERS]: exports.createUserBuildersView, + [ViewName.USER_BY_APP]: exports.createUserAppView, } // can pass DB in if working with something specific if (!db) { @@ -98,6 +148,7 @@ exports.queryGlobalView = async (viewName, params, db = null) => { } catch (err) { if (err != null && err.name === "not_found") { const createFunc = CreateFuncByName[viewName] + await removeDeprecated(db, viewName) await createFunc() return exports.queryGlobalView(viewName, params) } else { diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 0a17c8287..0348d921a 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -40,7 +40,7 @@ const env = { DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL, SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED || ""), COOKIE_DOMAIN: process.env.COOKIE_DOMAIN, - PLATFORM_URL: process.env.PLATFORM_URL, + PLATFORM_URL: process.env.PLATFORM_URL || "", POSTHOG_TOKEN: process.env.POSTHOG_TOKEN, ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS, TENANT_FEATURE_FLAGS: process.env.TENANT_FEATURE_FLAGS, @@ -54,6 +54,9 @@ const env = { DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE, DEFAULT_LICENSE: process.env.DEFAULT_LICENSE, SERVICE: process.env.SERVICE || "budibase", + MEMORY_LEAK_CHECK: process.env.MEMORY_LEAK_CHECK || false, + LOG_LEVEL: process.env.LOG_LEVEL, + SESSION_UPDATE_PERIOD: process.env.SESSION_UPDATE_PERIOD, DEPLOYMENT_ENVIRONMENT: process.env.DEPLOYMENT_ENVIRONMENT || "docker-compose", _set(key: any, value: any) { diff --git a/packages/backend-core/src/errors/index.js b/packages/backend-core/src/errors/index.js index 58b4eea8c..31ffd739a 100644 --- a/packages/backend-core/src/errors/index.js +++ b/packages/backend-core/src/errors/index.js @@ -37,6 +37,7 @@ module.exports = { types, errors: { UsageLimitError: licensing.UsageLimitError, + FeatureDisabledError: licensing.FeatureDisabledError, HTTPError: http.HTTPError, }, getPublicError, diff --git a/packages/backend-core/src/errors/licensing.js b/packages/backend-core/src/errors/licensing.js index 0d8ce0814..85d207ac3 100644 --- a/packages/backend-core/src/errors/licensing.js +++ b/packages/backend-core/src/errors/licensing.js @@ -4,6 +4,7 @@ const type = "license_error" const codes = { USAGE_LIMIT_EXCEEDED: "usage_limit_exceeded", + FEATURE_DISABLED: "feature_disabled", } const context = { @@ -12,6 +13,11 @@ const context = { limitName: err.limitName, } }, + [codes.FEATURE_DISABLED]: err => { + return { + featureName: err.featureName, + } + }, } class UsageLimitError extends HTTPError { @@ -21,9 +27,17 @@ class UsageLimitError extends HTTPError { } } +class FeatureDisabledError extends HTTPError { + constructor(message, featureName) { + super(message, 400, codes.FEATURE_DISABLED, type) + this.featureName = featureName + } +} + module.exports = { type, codes, context, UsageLimitError, + FeatureDisabledError, } diff --git a/packages/backend-core/src/events/processors/AnalyticsProcessor.ts b/packages/backend-core/src/events/processors/AnalyticsProcessor.ts index 2ee7a02af..f9d754712 100644 --- a/packages/backend-core/src/events/processors/AnalyticsProcessor.ts +++ b/packages/backend-core/src/events/processors/AnalyticsProcessor.ts @@ -2,7 +2,7 @@ import { Event, Identity, Group, IdentityType } from "@budibase/types" import { EventProcessor } from "./types" import env from "../../environment" import * as analytics from "../analytics" -import PosthogProcessor from "./PosthogProcessor" +import PosthogProcessor from "./posthog" /** * Events that are always captured. @@ -32,7 +32,7 @@ export default class AnalyticsProcessor implements EventProcessor { return } if (this.posthog) { - this.posthog.processEvent(event, identity, properties, timestamp) + await this.posthog.processEvent(event, identity, properties, timestamp) } } @@ -45,14 +45,14 @@ export default class AnalyticsProcessor implements EventProcessor { return } if (this.posthog) { - this.posthog.identify(identity, timestamp) + await this.posthog.identify(identity, timestamp) } } async identifyGroup(group: Group, timestamp?: string | number) { // Group indentifications (tenant and installation) always on if (this.posthog) { - this.posthog.identifyGroup(group, timestamp) + await this.posthog.identifyGroup(group, timestamp) } } diff --git a/packages/backend-core/src/events/processors/PosthogProcessor.ts b/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts similarity index 72% rename from packages/backend-core/src/events/processors/PosthogProcessor.ts rename to packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts index 67407fdd5..593e5ff08 100644 --- a/packages/backend-core/src/events/processors/PosthogProcessor.ts +++ b/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts @@ -1,9 +1,26 @@ import PostHog from "posthog-node" import { Event, Identity, Group, BaseEvent } from "@budibase/types" -import { EventProcessor } from "./types" -import env from "../../environment" -import context from "../../context" -const pkg = require("../../../package.json") +import { EventProcessor } from "../types" +import env from "../../../environment" +import * as context from "../../../context" +import * as rateLimiting from "./rateLimiting" +const pkg = require("../../../../package.json") + +const EXCLUDED_EVENTS: Event[] = [ + Event.USER_UPDATED, + Event.EMAIL_SMTP_UPDATED, + Event.AUTH_SSO_UPDATED, + Event.APP_UPDATED, + Event.ROLE_UPDATED, + Event.DATASOURCE_UPDATED, + Event.QUERY_UPDATED, + Event.TABLE_UPDATED, + Event.VIEW_UPDATED, + Event.VIEW_FILTER_UPDATED, + Event.VIEW_CALCULATION_UPDATED, + Event.AUTOMATION_TRIGGER_UPDATED, + Event.USER_GROUP_UPDATED, +] export default class PosthogProcessor implements EventProcessor { posthog: PostHog @@ -21,6 +38,15 @@ export default class PosthogProcessor implements EventProcessor { properties: BaseEvent, timestamp?: string | number ): Promise { + // don't send excluded events + if (EXCLUDED_EVENTS.includes(event)) { + return + } + + if (await rateLimiting.limited(event)) { + return + } + properties.version = pkg.version properties.service = env.SERVICE properties.environment = identity.environment diff --git a/packages/backend-core/src/events/processors/posthog/index.ts b/packages/backend-core/src/events/processors/posthog/index.ts new file mode 100644 index 000000000..dceb10d2c --- /dev/null +++ b/packages/backend-core/src/events/processors/posthog/index.ts @@ -0,0 +1,2 @@ +import PosthogProcessor from "./PosthogProcessor" +export default PosthogProcessor diff --git a/packages/backend-core/src/events/processors/posthog/rateLimiting.ts b/packages/backend-core/src/events/processors/posthog/rateLimiting.ts new file mode 100644 index 000000000..9c7b7876d --- /dev/null +++ b/packages/backend-core/src/events/processors/posthog/rateLimiting.ts @@ -0,0 +1,106 @@ +import { Event } from "@budibase/types" +import { CacheKeys, TTL } from "../../../cache/generic" +import * as cache from "../../../cache/generic" +import * as context from "../../../context" + +type RateLimitedEvent = + | Event.SERVED_BUILDER + | Event.SERVED_APP_PREVIEW + | Event.SERVED_APP + +const isRateLimited = (event: Event): event is RateLimitedEvent => { + return ( + event === Event.SERVED_BUILDER || + event === Event.SERVED_APP_PREVIEW || + event === Event.SERVED_APP + ) +} + +const isPerApp = (event: RateLimitedEvent) => { + return event === Event.SERVED_APP_PREVIEW || event === Event.SERVED_APP +} + +interface EventProperties { + timestamp: number +} + +enum RateLimit { + CALENDAR_DAY = "calendarDay", +} + +const RATE_LIMITS = { + [Event.SERVED_APP]: RateLimit.CALENDAR_DAY, + [Event.SERVED_APP_PREVIEW]: RateLimit.CALENDAR_DAY, + [Event.SERVED_BUILDER]: RateLimit.CALENDAR_DAY, +} + +/** + * Check if this event should be sent right now + * Return false to signal the event SHOULD be sent + * Return true to signal the event should NOT be sent + */ +export const limited = async (event: Event): Promise => { + // not a rate limited event -- send + if (!isRateLimited(event)) { + return false + } + + const cachedEvent = await readEvent(event) + if (cachedEvent) { + const timestamp = new Date(cachedEvent.timestamp) + const limit = RATE_LIMITS[event] + switch (limit) { + case RateLimit.CALENDAR_DAY: { + // get midnight at the start of the next day for the timestamp + timestamp.setDate(timestamp.getDate() + 1) + timestamp.setHours(0, 0, 0, 0) + + // if we have passed the threshold into the next day + if (Date.now() > timestamp.getTime()) { + // update the timestamp in the event -- send + await recordEvent(event, { timestamp: Date.now() }) + return false + } else { + // still within the limited period -- don't send + return true + } + } + } + } else { + // no event present i.e. expired -- send + await recordEvent(event, { timestamp: Date.now() }) + return false + } +} + +const eventKey = (event: RateLimitedEvent) => { + let key = `${CacheKeys.EVENTS_RATE_LIMIT}:${event}` + if (isPerApp(event)) { + key = key + ":" + context.getAppId() + } + return key +} + +const readEvent = async ( + event: RateLimitedEvent +): Promise => { + const key = eventKey(event) + const result = await cache.get(key) + return result as EventProperties +} + +const recordEvent = async ( + event: RateLimitedEvent, + properties: EventProperties +) => { + const key = eventKey(event) + const limit = RATE_LIMITS[event] + let ttl + switch (limit) { + case RateLimit.CALENDAR_DAY: { + ttl = TTL.ONE_DAY + } + } + + await cache.store(key, properties, ttl) +} diff --git a/packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts b/packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts new file mode 100644 index 000000000..d14b69796 --- /dev/null +++ b/packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts @@ -0,0 +1,145 @@ +import "../../../../../tests/utilities/TestConfiguration" +import PosthogProcessor from "../PosthogProcessor" +import { Event, IdentityType, Hosting } from "@budibase/types" +const tk = require("timekeeper") +import * as cache from "../../../../cache/generic" +import { CacheKeys } from "../../../../cache/generic" +import * as context from "../../../../context" + +const newIdentity = () => { + return { + id: "test", + type: IdentityType.USER, + hosting: Hosting.SELF, + environment: "test", + } +} + +describe("PosthogProcessor", () => { + beforeEach(async () => { + jest.clearAllMocks() + await cache.bustCache( + `${CacheKeys.EVENTS_RATE_LIMIT}:${Event.SERVED_BUILDER}` + ) + }) + + describe("processEvent", () => { + it("processes event", async () => { + const processor = new PosthogProcessor("test") + + const identity = newIdentity() + const properties = {} + + await processor.processEvent(Event.APP_CREATED, identity, properties) + + expect(processor.posthog.capture).toHaveBeenCalledTimes(1) + }) + + it("honours exclusions", async () => { + const processor = new PosthogProcessor("test") + + const identity = newIdentity() + const properties = {} + + await processor.processEvent(Event.AUTH_SSO_UPDATED, identity, properties) + expect(processor.posthog.capture).toHaveBeenCalledTimes(0) + }) + + describe("rate limiting", () => { + it("sends daily event once in same day", async () => { + const processor = new PosthogProcessor("test") + const identity = newIdentity() + const properties = {} + + tk.freeze(new Date(2022, 0, 1, 14, 0)) + await processor.processEvent(Event.SERVED_BUILDER, identity, properties) + // go forward one hour + tk.freeze(new Date(2022, 0, 1, 15, 0)) + await processor.processEvent(Event.SERVED_BUILDER, identity, properties) + + expect(processor.posthog.capture).toHaveBeenCalledTimes(1) + }) + + it("sends daily event once per unique day", async () => { + const processor = new PosthogProcessor("test") + const identity = newIdentity() + const properties = {} + + tk.freeze(new Date(2022, 0, 1, 14, 0)) + await processor.processEvent(Event.SERVED_BUILDER, identity, properties) + // go forward into next day + tk.freeze(new Date(2022, 0, 2, 9, 0)) + await processor.processEvent(Event.SERVED_BUILDER, identity, properties) + // go forward into next day + tk.freeze(new Date(2022, 0, 3, 5, 0)) + await processor.processEvent(Event.SERVED_BUILDER, identity, properties) + // go forward one hour + tk.freeze(new Date(2022, 0, 3, 6, 0)) + await processor.processEvent(Event.SERVED_BUILDER, identity, properties) + + expect(processor.posthog.capture).toHaveBeenCalledTimes(3) + }) + + it("sends event again after cache expires", async () => { + const processor = new PosthogProcessor("test") + const identity = newIdentity() + const properties = {} + + tk.freeze(new Date(2022, 0, 1, 14, 0)) + await processor.processEvent(Event.SERVED_BUILDER, identity, properties) + + await cache.bustCache( + `${CacheKeys.EVENTS_RATE_LIMIT}:${Event.SERVED_BUILDER}` + ) + + tk.freeze(new Date(2022, 0, 1, 14, 0)) + await processor.processEvent(Event.SERVED_BUILDER, identity, properties) + + expect(processor.posthog.capture).toHaveBeenCalledTimes(2) + }) + + it("sends per app events once per day per app", async () => { + const processor = new PosthogProcessor("test") + const identity = newIdentity() + const properties = {} + + const runAppEvents = async (appId: string) => { + await context.doInAppContext(appId, async () => { + tk.freeze(new Date(2022, 0, 1, 14, 0)) + await processor.processEvent(Event.SERVED_APP, identity, properties) + await processor.processEvent( + Event.SERVED_APP_PREVIEW, + identity, + properties + ) + + // go forward one hour - should be ignored + tk.freeze(new Date(2022, 0, 1, 15, 0)) + await processor.processEvent(Event.SERVED_APP, identity, properties) + await processor.processEvent( + Event.SERVED_APP_PREVIEW, + identity, + properties + ) + + // go forward into next day + tk.freeze(new Date(2022, 0, 2, 9, 0)) + + await processor.processEvent(Event.SERVED_APP, identity, properties) + await processor.processEvent( + Event.SERVED_APP_PREVIEW, + identity, + properties + ) + }) + } + + await runAppEvents("app_1") + expect(processor.posthog.capture).toHaveBeenCalledTimes(4) + + await runAppEvents("app_2") + expect(processor.posthog.capture).toHaveBeenCalledTimes(8) + }) + }) + }) +}) diff --git a/packages/backend-core/src/events/publishers/group.ts b/packages/backend-core/src/events/publishers/group.ts new file mode 100644 index 000000000..d30087372 --- /dev/null +++ b/packages/backend-core/src/events/publishers/group.ts @@ -0,0 +1,64 @@ +import { publishEvent } from "../events" +import { + Event, + UserGroup, + GroupCreatedEvent, + GroupDeletedEvent, + GroupUpdatedEvent, + GroupUsersAddedEvent, + GroupUsersDeletedEvent, + GroupAddedOnboardingEvent, + UserGroupRoles, +} from "@budibase/types" + +export async function created(group: UserGroup, timestamp?: number) { + const properties: GroupCreatedEvent = { + groupId: group._id as string, + } + await publishEvent(Event.USER_GROUP_CREATED, properties, timestamp) +} + +export async function updated(group: UserGroup) { + const properties: GroupUpdatedEvent = { + groupId: group._id as string, + } + await publishEvent(Event.USER_GROUP_UPDATED, properties) +} + +export async function deleted(group: UserGroup) { + const properties: GroupDeletedEvent = { + groupId: group._id as string, + } + await publishEvent(Event.USER_GROUP_DELETED, properties) +} + +export async function usersAdded(count: number, group: UserGroup) { + const properties: GroupUsersAddedEvent = { + count, + groupId: group._id as string, + } + await publishEvent(Event.USER_GROUP_USERS_ADDED, properties) +} + +export async function usersDeleted(emails: string[], group: UserGroup) { + const properties: GroupUsersDeletedEvent = { + count: emails.length, + groupId: group._id as string, + } + await publishEvent(Event.USER_GROUP_USERS_REMOVED, properties) +} + +export async function createdOnboarding(groupId: string) { + const properties: GroupAddedOnboardingEvent = { + groupId: groupId, + onboarding: true, + } + await publishEvent(Event.USER_GROUP_ONBOARDING, properties) +} + +export async function permissionsEdited(roles: UserGroupRoles) { + const properties: UserGroupRoles = { + ...roles, + } + await publishEvent(Event.USER_GROUP_PERMISSIONS_EDITED, properties) +} diff --git a/packages/backend-core/src/events/publishers/index.ts b/packages/backend-core/src/events/publishers/index.ts index 65785d4d8..57fd0bf8e 100644 --- a/packages/backend-core/src/events/publishers/index.ts +++ b/packages/backend-core/src/events/publishers/index.ts @@ -17,3 +17,4 @@ export * as user from "./user" export * as view from "./view" export * as installation from "./installation" export * as backfill from "./backfill" +export * as group from "./group" diff --git a/packages/backend-core/src/events/publishers/license.ts b/packages/backend-core/src/events/publishers/license.ts index 44dafd84c..1adc71652 100644 --- a/packages/backend-core/src/events/publishers/license.ts +++ b/packages/backend-core/src/events/publishers/license.ts @@ -20,12 +20,6 @@ export async function downgraded(license: License) { await publishEvent(Event.LICENSE_DOWNGRADED, properties) } -// TODO -export async function updated(license: License) { - const properties: LicenseUpdatedEvent = {} - await publishEvent(Event.LICENSE_UPDATED, properties) -} - // TODO export async function activated(license: License) { const properties: LicenseActivatedEvent = {} diff --git a/packages/backend-core/src/events/publishers/serve.ts b/packages/backend-core/src/events/publishers/serve.ts index 13afede02..128e0b9b1 100644 --- a/packages/backend-core/src/events/publishers/serve.ts +++ b/packages/backend-core/src/events/publishers/serve.ts @@ -7,22 +7,26 @@ import { AppServedEvent, } from "@budibase/types" -export async function servedBuilder() { - const properties: BuilderServedEvent = {} +export async function servedBuilder(timezone: string) { + const properties: BuilderServedEvent = { + timezone, + } await publishEvent(Event.SERVED_BUILDER, properties) } -export async function servedApp(app: App) { +export async function servedApp(app: App, timezone: string) { const properties: AppServedEvent = { appVersion: app.version, + timezone, } await publishEvent(Event.SERVED_APP, properties) } -export async function servedAppPreview(app: App) { +export async function servedAppPreview(app: App, timezone: string) { const properties: AppPreviewServedEvent = { appId: app.appId, appVersion: app.version, + timezone, } await publishEvent(Event.SERVED_APP_PREVIEW, properties) } diff --git a/packages/backend-core/src/featureFlags/index.js b/packages/backend-core/src/featureFlags/index.js index c050cbdfe..103ac4df5 100644 --- a/packages/backend-core/src/featureFlags/index.js +++ b/packages/backend-core/src/featureFlags/index.js @@ -50,4 +50,5 @@ exports.getTenantFeatureFlags = tenantId => { exports.FeatureFlag = { LICENSING: "LICENSING", GOOGLE_SHEETS: "GOOGLE_SHEETS", + USER_GROUPS: "USER_GROUPS", } diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 6eb6b14bc..6d2e8dcd1 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -1,18 +1,22 @@ import errors from "./errors" + const errorClasses = errors.errors import * as events from "./events" import * as migrations from "./migrations" import * as users from "./users" +import * as roles from "./security/roles" import * as accounts from "./cloud/accounts" import * as installation from "./installation" import env from "./environment" import tenancy from "./tenancy" import featureFlags from "./featureFlags" -import sessions from "./security/sessions" +import * as sessions from "./security/sessions" import deprovisioning from "./context/deprovision" import auth from "./auth" import constants from "./constants" import * as dbConstants from "./db/constants" +import logging from "./logging" +import pino from "./pino" // mimic the outer package exports import * as db from "./pkg/db" @@ -49,6 +53,9 @@ const core = { deprovisioning, installation, errors, + logging, + roles, + ...pino, ...errorClasses, } diff --git a/packages/backend-core/src/logging.js b/packages/backend-core/src/logging.js deleted file mode 100644 index 6ab5bff44..000000000 --- a/packages/backend-core/src/logging.js +++ /dev/null @@ -1,16 +0,0 @@ -const NonErrors = ["AccountError"] - -function isSuppressed(e) { - return e && e["suppressAlert"] -} - -module.exports.logAlert = (message, e) => { - if (e && NonErrors.includes(e.name) && isSuppressed(e)) { - return - } - let errorJson = "" - if (e) { - errorJson = ": " + JSON.stringify(e, Object.getOwnPropertyNames(e)) - } - console.error(`bb-alert: ${message} ${errorJson}`) -} diff --git a/packages/backend-core/src/logging.ts b/packages/backend-core/src/logging.ts new file mode 100644 index 000000000..3fc79a5fe --- /dev/null +++ b/packages/backend-core/src/logging.ts @@ -0,0 +1,36 @@ +const NonErrors = ["AccountError"] + +function isSuppressed(e?: any) { + return e && e["suppressAlert"] +} + +export function logAlert(message: string, e?: any) { + if (e && NonErrors.includes(e.name) && isSuppressed(e)) { + return + } + let errorJson = "" + if (e) { + errorJson = ": " + JSON.stringify(e, Object.getOwnPropertyNames(e)) + } + console.error(`bb-alert: ${message} ${errorJson}`) +} + +export function logAlertWithInfo( + message: string, + db: string, + id: string, + error: any +) { + message = `${message} - db: ${db} - doc: ${id} - error: ` + logAlert(message, error) +} + +export function logWarn(message: string) { + console.warn(`bb-warn: ${message}`) +} + +export default { + logAlert, + logAlertWithInfo, + logWarn, +} diff --git a/packages/worker/src/middleware/adminOnly.js b/packages/backend-core/src/middleware/adminOnly.js similarity index 100% rename from packages/worker/src/middleware/adminOnly.js rename to packages/backend-core/src/middleware/adminOnly.js diff --git a/packages/backend-core/src/middleware/authenticated.js b/packages/backend-core/src/middleware/authenticated.ts similarity index 59% rename from packages/backend-core/src/middleware/authenticated.js rename to packages/backend-core/src/middleware/authenticated.ts index 9c35336dd..b51ead46b 100644 --- a/packages/backend-core/src/middleware/authenticated.js +++ b/packages/backend-core/src/middleware/authenticated.ts @@ -1,28 +1,39 @@ -const { Cookies, Headers } = require("../constants") -const { getCookie, clearCookie, openJwt } = require("../utils") -const { getUser } = require("../cache/user") -const { getSession, updateSessionTTL } = require("../security/sessions") -const { buildMatcherRegex, matches } = require("./matchers") -const env = require("../environment") -const { SEPARATOR } = require("../db/constants") -const { ViewNames } = require("../db/utils") -const { queryGlobalView } = require("../db/views") -const { getGlobalDB, doInTenant } = require("../tenancy") -const { decrypt } = require("../security/encryption") +import { Cookies, Headers } from "../constants" +import { getCookie, clearCookie, openJwt } from "../utils" +import { getUser } from "../cache/user" +import { getSession, updateSessionTTL } from "../security/sessions" +import { buildMatcherRegex, matches } from "./matchers" +import { SEPARATOR } from "../db/constants" +import { ViewName } from "../db/utils" +import { queryGlobalView } from "../db/views" +import { getGlobalDB, doInTenant } from "../tenancy" +import { decrypt } from "../security/encryption" const identity = require("../context/identity") +const env = require("../environment") + +const ONE_MINUTE = env.SESSION_UPDATE_PERIOD || 60 * 1000 + +interface FinaliseOpts { + authenticated?: boolean + internal?: boolean + publicEndpoint?: boolean + version?: string + user?: any +} + +function timeMinusOneMinute() { + return new Date(Date.now() - ONE_MINUTE).toISOString() +} -function finalise( - ctx, - { authenticated, user, internal, version, publicEndpoint } = {} -) { - ctx.publicEndpoint = publicEndpoint || false - ctx.isAuthenticated = authenticated || false - ctx.user = user - ctx.internal = internal || false - ctx.version = version +function finalise(ctx: any, opts: FinaliseOpts = {}) { + ctx.publicEndpoint = opts.publicEndpoint || false + ctx.isAuthenticated = opts.authenticated || false + ctx.user = opts.user + ctx.internal = opts.internal || false + ctx.version = opts.version } -async function checkApiKey(apiKey, populateUser) { +async function checkApiKey(apiKey: string, populateUser?: Function) { if (apiKey === env.INTERNAL_API_KEY) { return { valid: true } } @@ -32,7 +43,7 @@ async function checkApiKey(apiKey, populateUser) { const db = getGlobalDB() // api key is encrypted in the database const userId = await queryGlobalView( - ViewNames.BY_API_KEY, + ViewName.BY_API_KEY, { key: apiKey, }, @@ -56,10 +67,12 @@ async function checkApiKey(apiKey, populateUser) { */ module.exports = ( noAuthPatterns = [], - opts = { publicAllowed: false, populateUser: null } + opts: { publicAllowed: boolean; populateUser?: Function } = { + publicAllowed: false, + } ) => { const noAuthOptions = noAuthPatterns ? buildMatcherRegex(noAuthPatterns) : [] - return async (ctx, next) => { + return async (ctx: any, next: any) => { let publicEndpoint = false const version = ctx.request.headers[Headers.API_VER] // the path is not authenticated @@ -71,46 +84,40 @@ module.exports = ( // check the actual user is authenticated first, try header or cookie const headerToken = ctx.request.headers[Headers.TOKEN] const authCookie = getCookie(ctx, Cookies.Auth) || openJwt(headerToken) + const apiKey = ctx.request.headers[Headers.API_KEY] + const tenantId = ctx.request.headers[Headers.TENANT_ID] let authenticated = false, user = null, internal = false - if (authCookie) { - let error = null + if (authCookie && !apiKey) { const sessionId = authCookie.sessionId const userId = authCookie.userId - - const session = await getSession(userId, sessionId) - if (!session) { - error = "No session found" - } else { - try { - if (opts && opts.populateUser) { - user = await getUser( - userId, - session.tenantId, - opts.populateUser(ctx) - ) - } else { - user = await getUser(userId, session.tenantId) - } - user.csrfToken = session.csrfToken - delete user.password - authenticated = true - } catch (err) { - error = err + let session + try { + // getting session handles error checking (if session exists etc) + session = await getSession(userId, sessionId) + if (opts && opts.populateUser) { + user = await getUser( + userId, + session.tenantId, + opts.populateUser(ctx) + ) + } else { + user = await getUser(userId, session.tenantId) } - } - if (error) { - console.error("Auth Error", error) + user.csrfToken = session.csrfToken + if (session?.lastAccessedAt < timeMinusOneMinute()) { + // make sure we denote that the session is still in use + await updateSessionTTL(session) + } + authenticated = true + } catch (err: any) { + authenticated = false + console.error("Auth Error", err?.message || err) // remove the cookie as the user does not exist anymore clearCookie(ctx, Cookies.Auth) - } else { - // make sure we denote that the session is still in use - await updateSessionTTL(session) } } - const apiKey = ctx.request.headers[Headers.API_KEY] - const tenantId = ctx.request.headers[Headers.TENANT_ID] // this is an internal request, no user made it if (!authenticated && apiKey) { const populateUser = opts.populateUser ? opts.populateUser(ctx) : null @@ -128,6 +135,8 @@ module.exports = ( } if (!user && tenantId) { user = { tenantId } + } else if (user) { + delete user.password } // be explicit if (authenticated !== true) { @@ -141,7 +150,7 @@ module.exports = ( } else { return next() } - } catch (err) { + } catch (err: any) { // invalid token, clear the cookie if (err && err.name === "JsonWebTokenError") { clearCookie(ctx, Cookies.Auth) diff --git a/packages/worker/src/middleware/builderOnly.js b/packages/backend-core/src/middleware/builderOnly.js similarity index 100% rename from packages/worker/src/middleware/builderOnly.js rename to packages/backend-core/src/middleware/builderOnly.js diff --git a/packages/worker/src/middleware/builderOrAdmin.js b/packages/backend-core/src/middleware/builderOrAdmin.js similarity index 100% rename from packages/worker/src/middleware/builderOrAdmin.js rename to packages/backend-core/src/middleware/builderOrAdmin.js diff --git a/packages/backend-core/src/middleware/index.js b/packages/backend-core/src/middleware/index.js index 6c4c0d888..7e7b8a293 100644 --- a/packages/backend-core/src/middleware/index.js +++ b/packages/backend-core/src/middleware/index.js @@ -2,14 +2,17 @@ const jwt = require("./passport/jwt") const local = require("./passport/local") const google = require("./passport/google") const oidc = require("./passport/oidc") -const { authError } = require("./passport/utils") +const { authError, ssoCallbackUrl } = require("./passport/utils") const authenticated = require("./authenticated") const auditLog = require("./auditLog") const tenancy = require("./tenancy") const internalApi = require("./internalApi") const datasourceGoogle = require("./passport/datasource/google") const csrf = require("./csrf") - +const adminOnly = require("./adminOnly") +const builderOrAdmin = require("./builderOrAdmin") +const builderOnly = require("./builderOnly") +const joiValidator = require("./joi-validator") module.exports = { google, oidc, @@ -20,8 +23,13 @@ module.exports = { tenancy, authError, internalApi, + ssoCallbackUrl, datasource: { google: datasourceGoogle, }, csrf, + adminOnly, + builderOnly, + builderOrAdmin, + joiValidator, } diff --git a/packages/worker/src/middleware/joi-validator.js b/packages/backend-core/src/middleware/joi-validator.js similarity index 81% rename from packages/worker/src/middleware/joi-validator.js rename to packages/backend-core/src/middleware/joi-validator.js index 1686b0e72..748ccebd8 100644 --- a/packages/worker/src/middleware/joi-validator.js +++ b/packages/backend-core/src/middleware/joi-validator.js @@ -1,3 +1,5 @@ +const Joi = require("joi") + function validate(schema, property) { // Return a Koa middleware function return (ctx, next) => { @@ -10,6 +12,12 @@ function validate(schema, property) { } else if (ctx.request[property] != null) { params = ctx.request[property] } + + schema = schema.append({ + createdAt: Joi.any().optional(), + updatedAt: Joi.any().optional(), + }) + const { error } = schema.validate(params) if (error) { ctx.throw(400, `Invalid ${property} - ${error.message}`) diff --git a/packages/backend-core/src/middleware/passport/google.js b/packages/backend-core/src/middleware/passport/google.js index 858029ca8..7419974cd 100644 --- a/packages/backend-core/src/middleware/passport/google.js +++ b/packages/backend-core/src/middleware/passport/google.js @@ -1,6 +1,7 @@ const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy - +const { ssoCallbackUrl } = require("./utils") const { authenticateThirdParty } = require("./third-party-common") +const { Configs } = require("../../../constants") const buildVerifyFn = saveUserFn => { return (accessToken, refreshToken, profile, done) => { @@ -57,5 +58,10 @@ exports.strategyFactory = async function (config, callbackUrl, saveUserFn) { ) } } + +exports.getCallbackUrl = async function (db, config) { + return ssoCallbackUrl(db, config, Configs.GOOGLE) +} + // expose for testing exports.buildVerifyFn = buildVerifyFn diff --git a/packages/backend-core/src/middleware/passport/local.js b/packages/backend-core/src/middleware/passport/local.js index 445893b1d..b955d2910 100644 --- a/packages/backend-core/src/middleware/passport/local.js +++ b/packages/backend-core/src/middleware/passport/local.js @@ -55,6 +55,7 @@ exports.authenticate = async function (ctx, email, password, done) { if (await compare(password, dbUser.password)) { const sessionId = newid() const tenantId = getTenantId() + await createASession(dbUser._id, { sessionId, tenantId }) dbUser.token = jwt.sign( diff --git a/packages/backend-core/src/middleware/passport/oidc.js b/packages/backend-core/src/middleware/passport/oidc.js index 1e93e20b1..20dbd4669 100644 --- a/packages/backend-core/src/middleware/passport/oidc.js +++ b/packages/backend-core/src/middleware/passport/oidc.js @@ -1,6 +1,8 @@ const fetch = require("node-fetch") const OIDCStrategy = require("@techpass/passport-openidconnect").Strategy const { authenticateThirdParty } = require("./third-party-common") +const { ssoCallbackUrl } = require("./utils") +const { Configs } = require("../../../constants") const buildVerifyFn = saveUserFn => { /** @@ -89,11 +91,24 @@ function validEmail(value) { * from couchDB rather than environment variables, using this factory is necessary for dynamically configuring passport. * @returns Dynamically configured Passport OIDC Strategy */ -exports.strategyFactory = async function (config, callbackUrl, saveUserFn) { +exports.strategyFactory = async function (config, saveUserFn) { try { - const { clientID, clientSecret, configUrl } = config + const verify = buildVerifyFn(saveUserFn) + const strategy = new OIDCStrategy(config, verify) + strategy.name = "oidc" + return strategy + } catch (err) { + console.error(err) + throw new Error("Error constructing OIDC authentication strategy", err) + } +} + +exports.fetchStrategyConfig = async function (enrichedConfig, callbackUrl) { + try { + const { clientID, clientSecret, configUrl } = enrichedConfig if (!clientID || !clientSecret || !callbackUrl || !configUrl) { + //check for remote config and all required elements throw new Error( "Configuration invalid. Must contain clientID, clientSecret, callbackUrl and configUrl" ) @@ -109,24 +124,24 @@ exports.strategyFactory = async function (config, callbackUrl, saveUserFn) { const body = await response.json() - const verify = buildVerifyFn(saveUserFn) - return new OIDCStrategy( - { - issuer: body.issuer, - authorizationURL: body.authorization_endpoint, - tokenURL: body.token_endpoint, - userInfoURL: body.userinfo_endpoint, - clientID: clientID, - clientSecret: clientSecret, - callbackURL: callbackUrl, - }, - verify - ) + return { + issuer: body.issuer, + authorizationURL: body.authorization_endpoint, + tokenURL: body.token_endpoint, + userInfoURL: body.userinfo_endpoint, + clientID: clientID, + clientSecret: clientSecret, + callbackURL: callbackUrl, + } } catch (err) { console.error(err) - throw new Error("Error constructing OIDC authentication strategy", err) + throw new Error("Error constructing OIDC authentication configuration", err) } } +exports.getCallbackUrl = async function (db, config) { + return ssoCallbackUrl(db, config, Configs.OIDC) +} + // expose for testing exports.buildVerifyFn = buildVerifyFn diff --git a/packages/backend-core/src/middleware/passport/tests/oidc.spec.js b/packages/backend-core/src/middleware/passport/tests/oidc.spec.js index c5e9fe003..c00ab2ea7 100644 --- a/packages/backend-core/src/middleware/passport/tests/oidc.spec.js +++ b/packages/backend-core/src/middleware/passport/tests/oidc.spec.js @@ -48,8 +48,8 @@ describe("oidc", () => { it("should create successfully create an oidc strategy", async () => { const oidc = require("../oidc") - - await oidc.strategyFactory(oidcConfig, callbackUrl) + const enrichedConfig = await oidc.fetchStrategyConfig(oidcConfig, callbackUrl) + await oidc.strategyFactory(enrichedConfig, callbackUrl) expect(mockFetch).toHaveBeenCalledWith(oidcConfig.configUrl) diff --git a/packages/backend-core/src/middleware/passport/utils.js b/packages/backend-core/src/middleware/passport/utils.js index cbb93bfa3..217130cd6 100644 --- a/packages/backend-core/src/middleware/passport/utils.js +++ b/packages/backend-core/src/middleware/passport/utils.js @@ -1,3 +1,7 @@ +const { isMultiTenant, getTenantId } = require("../../tenancy") +const { getScopedConfig } = require("../../db/utils") +const { Configs } = require("../../constants") + /** * Utility to handle authentication errors. * @@ -5,6 +9,7 @@ * @param {*} message Message that will be returned in the response body * @param {*} err (Optional) error that will be logged */ + exports.authError = function (done, message, err = null) { return done( err, @@ -12,3 +17,21 @@ exports.authError = function (done, message, err = null) { { message: message } ) } + +exports.ssoCallbackUrl = async (db, config, type) => { + // incase there is a callback URL from before + if (config && config.callbackURL) { + return config.callbackURL + } + const publicConfig = await getScopedConfig(db, { + type: Configs.SETTINGS, + }) + + let callbackUrl = `/api/global/auth` + if (isMultiTenant()) { + callbackUrl += `/${getTenantId()}` + } + callbackUrl += `/${type}/callback` + + return `${publicConfig.platformUrl}${callbackUrl}` +} diff --git a/packages/backend-core/src/migrations/definitions.ts b/packages/backend-core/src/migrations/definitions.ts index 745c8718c..34ec0f0ca 100644 --- a/packages/backend-core/src/migrations/definitions.ts +++ b/packages/backend-core/src/migrations/definitions.ts @@ -37,4 +37,8 @@ export const DEFINITIONS: MigrationDefinition[] = [ type: MigrationType.INSTALLATION, name: MigrationName.EVENT_INSTALLATION_BACKFILL, }, + { + type: MigrationType.GLOBAL, + name: MigrationName.GLOBAL_INFO_SYNC_USERS, + }, ] diff --git a/packages/backend-core/src/migrations/migrations.ts b/packages/backend-core/src/migrations/migrations.ts index b92633431..ca238ff80 100644 --- a/packages/backend-core/src/migrations/migrations.ts +++ b/packages/backend-core/src/migrations/migrations.ts @@ -1,6 +1,6 @@ import { DEFAULT_TENANT_ID } from "../constants" import { doWithDB } from "../db" -import { DocumentTypes, StaticDatabases } from "../db/constants" +import { DocumentType, StaticDatabases } from "../db/constants" import { getAllApps } from "../db/utils" import environment from "../environment" import { @@ -9,7 +9,7 @@ import { getGlobalDBName, getTenantId, } from "../tenancy" -import context from "../context" +import * as context from "../context" import { DEFINITIONS } from "." import { Migration, @@ -21,10 +21,10 @@ import { export const getMigrationsDoc = async (db: any) => { // get the migrations doc try { - return await db.get(DocumentTypes.MIGRATIONS) + return await db.get(DocumentType.MIGRATIONS) } catch (err: any) { if (err.status && err.status === 404) { - return { _id: DocumentTypes.MIGRATIONS } + return { _id: DocumentType.MIGRATIONS } } else { console.error(err) throw err diff --git a/packages/backend-core/src/objectStore/index.ts b/packages/backend-core/src/objectStore/index.ts index a7e0b0c13..503ab9bca 100644 --- a/packages/backend-core/src/objectStore/index.ts +++ b/packages/backend-core/src/objectStore/index.ts @@ -75,9 +75,11 @@ export const ObjectStore = (bucket: any) => { s3ForcePathStyle: true, signatureVersion: "v4", apiVersion: "2006-03-01", - params: { + } + if (bucket) { + config.params = { Bucket: sanitizeBucket(bucket), - }, + } } if (env.MINIO_URL) { config.endpoint = env.MINIO_URL @@ -292,6 +294,7 @@ export const uploadDirectory = async ( } } await Promise.all(uploads) + return files } exports.downloadTarballDirect = async (url: string, path: string) => { diff --git a/packages/backend-core/src/pino.js b/packages/backend-core/src/pino.js new file mode 100644 index 000000000..69962b384 --- /dev/null +++ b/packages/backend-core/src/pino.js @@ -0,0 +1,11 @@ +const env = require("./environment") + +exports.pinoSettings = () => ({ + prettyPrint: { + levelFirst: true, + }, + level: env.LOG_LEVEL || "error", + autoLogging: { + ignore: req => req.url.includes("/health"), + }, +}) diff --git a/packages/backend-core/src/security/roles.js b/packages/backend-core/src/security/roles.js index 7c57cadcb..983aebf67 100644 --- a/packages/backend-core/src/security/roles.js +++ b/packages/backend-core/src/security/roles.js @@ -3,7 +3,7 @@ const { BUILTIN_PERMISSION_IDS, PermissionLevels } = require("./permissions") const { generateRoleID, getRoleParams, - DocumentTypes, + DocumentType, SEPARATOR, } = require("../db/utils") const { getAppDB } = require("../context") @@ -76,7 +76,7 @@ function isBuiltin(role) { /** * Works through the inheritance ranks to see how far up the builtin stack this ID is. */ -function builtinRoleToNumber(id) { +exports.builtinRoleToNumber = id => { const builtins = exports.getBuiltinRoles() const MAX = Object.values(BUILTIN_IDS).length + 1 if (id === BUILTIN_IDS.ADMIN || id === BUILTIN_IDS.BUILDER) { @@ -104,7 +104,8 @@ exports.lowerBuiltinRoleID = (roleId1, roleId2) => { if (!roleId2) { return roleId1 } - return builtinRoleToNumber(roleId1) > builtinRoleToNumber(roleId2) + return exports.builtinRoleToNumber(roleId1) > + exports.builtinRoleToNumber(roleId2) ? roleId2 : roleId1 } @@ -202,15 +203,24 @@ exports.getAllRoles = async appId => { if (appId) { return doWithDB(appId, internal) } else { - return internal(getAppDB()) + let appDB + try { + appDB = getAppDB() + } catch (error) { + // We don't have any apps, so we'll just use the built-in roles + } + return internal(appDB) } async function internal(db) { - const body = await db.allDocs( - getRoleParams(null, { - include_docs: true, - }) - ) - let roles = body.rows.map(row => row.doc) + let roles = [] + if (db) { + const body = await db.allDocs( + getRoleParams(null, { + include_docs: true, + }) + ) + roles = body.rows.map(row => row.doc) + } const builtinRoles = exports.getBuiltinRoles() // need to combine builtin with any DB record of them (for sake of permissions) @@ -328,7 +338,7 @@ class AccessController { * Adds the "role_" for builtin role IDs which are to be written to the DB (for permissions). */ exports.getDBRoleID = roleId => { - if (roleId.startsWith(DocumentTypes.ROLE)) { + if (roleId.startsWith(DocumentType.ROLE)) { return roleId } return generateRoleID(roleId) @@ -339,8 +349,8 @@ exports.getDBRoleID = roleId => { */ exports.getExternalRoleID = roleId => { // for built in roles we want to remove the DB role ID element (role_) - if (roleId.startsWith(DocumentTypes.ROLE) && isBuiltin(roleId)) { - return roleId.split(`${DocumentTypes.ROLE}${SEPARATOR}`)[1] + if (roleId.startsWith(DocumentType.ROLE) && isBuiltin(roleId)) { + return roleId.split(`${DocumentType.ROLE}${SEPARATOR}`)[1] } return roleId } diff --git a/packages/backend-core/src/security/sessions.js b/packages/backend-core/src/security/sessions.js deleted file mode 100644 index 8874b4746..000000000 --- a/packages/backend-core/src/security/sessions.js +++ /dev/null @@ -1,95 +0,0 @@ -const redis = require("../redis/init") -const { v4: uuidv4 } = require("uuid") - -// a week in seconds -const EXPIRY_SECONDS = 86400 * 7 - -async function getSessionsForUser(userId) { - const client = await redis.getSessionClient() - const sessions = await client.scan(userId) - return sessions.map(session => session.value) -} - -function makeSessionID(userId, sessionId) { - return `${userId}/${sessionId}` -} - -async function invalidateSessions(userId, sessionIds = null) { - try { - let sessions = [] - - // If no sessionIds, get all the sessions for the user - if (!sessionIds) { - sessions = await getSessionsForUser(userId) - sessions.forEach( - session => - (session.key = makeSessionID(session.userId, session.sessionId)) - ) - } else { - // use the passed array of sessionIds - sessions = Array.isArray(sessionIds) ? sessionIds : [sessionIds] - sessions = sessions.map(sessionId => ({ - key: makeSessionID(userId, sessionId), - })) - } - - const client = await redis.getSessionClient() - const promises = [] - for (let session of sessions) { - promises.push(client.delete(session.key)) - } - await Promise.all(promises) - } catch (err) { - console.error(`Error invalidating sessions: ${err}`) - } -} - -exports.createASession = async (userId, session) => { - // invalidate all other sessions - await invalidateSessions(userId) - - const client = await redis.getSessionClient() - const sessionId = session.sessionId - if (!session.csrfToken) { - session.csrfToken = uuidv4() - } - session = { - createdAt: new Date().toISOString(), - lastAccessedAt: new Date().toISOString(), - ...session, - userId, - } - await client.store(makeSessionID(userId, sessionId), session, EXPIRY_SECONDS) -} - -exports.updateSessionTTL = async session => { - const client = await redis.getSessionClient() - const key = makeSessionID(session.userId, session.sessionId) - session.lastAccessedAt = new Date().toISOString() - await client.store(key, session, EXPIRY_SECONDS) -} - -exports.endSession = async (userId, sessionId) => { - const client = await redis.getSessionClient() - await client.delete(makeSessionID(userId, sessionId)) -} - -exports.getSession = async (userId, sessionId) => { - try { - const client = await redis.getSessionClient() - return client.get(makeSessionID(userId, sessionId)) - } catch (err) { - // if can't get session don't error, just don't return anything - console.error(err) - return null - } -} - -exports.getAllSessions = async () => { - const client = await redis.getSessionClient() - const sessions = await client.scan() - return sessions.map(session => session.value) -} - -exports.getUserSessions = getSessionsForUser -exports.invalidateSessions = invalidateSessions diff --git a/packages/backend-core/src/security/sessions.ts b/packages/backend-core/src/security/sessions.ts new file mode 100644 index 000000000..284adbcd1 --- /dev/null +++ b/packages/backend-core/src/security/sessions.ts @@ -0,0 +1,119 @@ +const redis = require("../redis/init") +const { v4: uuidv4 } = require("uuid") +const { logWarn } = require("../logging") +const env = require("../environment") + +interface Session { + key: string + userId: string + sessionId: string + lastAccessedAt: string + createdAt: string + csrfToken?: string + value: string +} + +type SessionKey = { key: string }[] + +// a week in seconds +const EXPIRY_SECONDS = 86400 * 7 + +function makeSessionID(userId: string, sessionId: string) { + return `${userId}/${sessionId}` +} + +export async function getSessionsForUser(userId: string) { + if (!userId) { + console.trace("Cannot get sessions for undefined userId") + return [] + } + const client = await redis.getSessionClient() + const sessions = await client.scan(userId) + return sessions.map((session: Session) => session.value) +} + +export async function invalidateSessions( + userId: string, + opts: { sessionIds?: string[]; reason?: string } = {} +) { + try { + const reason = opts?.reason || "unknown" + let sessionIds: string[] = opts.sessionIds || [] + let sessions: SessionKey + + // If no sessionIds, get all the sessions for the user + if (sessionIds.length === 0) { + sessions = await getSessionsForUser(userId) + sessions.forEach( + (session: any) => + (session.key = makeSessionID(session.userId, session.sessionId)) + ) + } else { + // use the passed array of sessionIds + sessionIds = Array.isArray(sessionIds) ? sessionIds : [sessionIds] + sessions = sessionIds.map((sessionId: string) => ({ + key: makeSessionID(userId, sessionId), + })) + } + + if (sessions && sessions.length > 0) { + const client = await redis.getSessionClient() + const promises = [] + for (let session of sessions) { + promises.push(client.delete(session.key)) + } + if (!env.isTest()) { + logWarn( + `Invalidating sessions for ${userId} (reason: ${reason}) - ${sessions + .map(session => session.key) + .join(", ")}` + ) + } + await Promise.all(promises) + } + } catch (err) { + console.error(`Error invalidating sessions: ${err}`) + } +} + +export async function createASession(userId: string, session: Session) { + // invalidate all other sessions + await invalidateSessions(userId, { reason: "creation" }) + + const client = await redis.getSessionClient() + const sessionId = session.sessionId + if (!session.csrfToken) { + session.csrfToken = uuidv4() + } + session = { + ...session, + createdAt: new Date().toISOString(), + lastAccessedAt: new Date().toISOString(), + userId, + } + await client.store(makeSessionID(userId, sessionId), session, EXPIRY_SECONDS) +} + +export async function updateSessionTTL(session: Session) { + const client = await redis.getSessionClient() + const key = makeSessionID(session.userId, session.sessionId) + session.lastAccessedAt = new Date().toISOString() + await client.store(key, session, EXPIRY_SECONDS) +} + +export async function endSession(userId: string, sessionId: string) { + const client = await redis.getSessionClient() + await client.delete(makeSessionID(userId, sessionId)) +} + +export async function getSession(userId: string, sessionId: string) { + if (!userId || !sessionId) { + throw new Error(`Invalid session details - ${userId} - ${sessionId}`) + } + const client = await redis.getSessionClient() + const session = await client.get(makeSessionID(userId, sessionId)) + if (!session) { + throw new Error(`Session not found - ${userId} - ${sessionId}`) + } + return session +} diff --git a/packages/backend-core/src/security/tests/sessions.spec.ts b/packages/backend-core/src/security/tests/sessions.spec.ts new file mode 100644 index 000000000..7f01bdcdb --- /dev/null +++ b/packages/backend-core/src/security/tests/sessions.spec.ts @@ -0,0 +1,12 @@ +import * as sessions from "../sessions" + +describe("sessions", () => { + describe("getSessionsForUser", () => { + it("returns empty when user is undefined", async () => { + // @ts-ignore - allow the undefined to be passed + const results = await sessions.getSessionsForUser(undefined) + + expect(results).toStrictEqual([]) + }) + }) +}) diff --git a/packages/backend-core/src/users.js b/packages/backend-core/src/users.js index 4acccda2a..de5ce238c 100644 --- a/packages/backend-core/src/users.js +++ b/packages/backend-core/src/users.js @@ -1,5 +1,11 @@ -const { ViewNames } = require("./db/utils") +const { + ViewName, + getUsersByAppParams, + getProdAppID, + generateAppUserID, +} = require("./db/utils") const { queryGlobalView } = require("./db/views") +const { UNICODE_MAX } = require("./db/constants") /** * Given an email address this will use a view to search through @@ -12,10 +18,51 @@ exports.getGlobalUserByEmail = async email => { throw "Must supply an email address to view" } - const response = await queryGlobalView(ViewNames.USER_BY_EMAIL, { + return await queryGlobalView(ViewName.USER_BY_EMAIL, { key: email.toLowerCase(), include_docs: true, }) +} + +exports.searchGlobalUsersByApp = async (appId, opts) => { + if (typeof appId !== "string") { + throw new Error("Must provide a string based app ID") + } + const params = getUsersByAppParams(appId, { + include_docs: true, + }) + params.startkey = opts && opts.startkey ? opts.startkey : params.startkey + let response = await queryGlobalView(ViewName.USER_BY_APP, params) + if (!response) { + response = [] + } + return Array.isArray(response) ? response : [response] +} - return response +exports.getGlobalUserByAppPage = (appId, user) => { + if (!user) { + return + } + return generateAppUserID(getProdAppID(appId), user._id) +} + +/** + * Performs a starts with search on the global email view. + */ +exports.searchGlobalUsersByEmail = async (email, opts) => { + if (typeof email !== "string") { + throw new Error("Must provide a string to search by") + } + const lcEmail = email.toLowerCase() + // handle if passing up startkey for pagination + const startkey = opts && opts.startkey ? opts.startkey : lcEmail + let response = await queryGlobalView(ViewName.USER_BY_EMAIL, { + ...opts, + startkey, + endkey: `${lcEmail}${UNICODE_MAX}`, + }) + if (!response) { + response = [] + } + return Array.isArray(response) ? response : [response] } diff --git a/packages/backend-core/src/utils.js b/packages/backend-core/src/utils.js index cf32539c5..0587267e9 100644 --- a/packages/backend-core/src/utils.js +++ b/packages/backend-core/src/utils.js @@ -1,20 +1,18 @@ -const { - DocumentTypes, - SEPARATOR, - ViewNames, - getAllApps, -} = require("./db/utils") +const { DocumentType, SEPARATOR, ViewName, getAllApps } = require("./db/utils") const jwt = require("jsonwebtoken") const { options } = require("./middleware/passport/jwt") const { queryGlobalView } = require("./db/views") const { Headers, Cookies, MAX_VALID_DATE } = require("./constants") const env = require("./environment") const userCache = require("./cache/user") -const { getUserSessions, invalidateSessions } = require("./security/sessions") +const { + getSessionsForUser, + invalidateSessions, +} = require("./security/sessions") const events = require("./events") const tenancy = require("./tenancy") -const APP_PREFIX = DocumentTypes.APP + SEPARATOR +const APP_PREFIX = DocumentType.APP + SEPARATOR const PROD_APP_PREFIX = "/app/" function confirmAppId(possibleAppId) { @@ -151,7 +149,7 @@ exports.isClient = ctx => { } const getBuilders = async () => { - const builders = await queryGlobalView(ViewNames.USER_BY_BUILDERS, { + const builders = await queryGlobalView(ViewName.USER_BY_BUILDERS, { include_docs: false, }) @@ -178,7 +176,7 @@ exports.platformLogout = async ({ ctx, userId, keepActiveSession }) => { if (!ctx) throw new Error("Koa context must be supplied to logout.") const currentSession = exports.getCookie(ctx, Cookies.Auth) - let sessions = await getUserSessions(userId) + let sessions = await getSessionsForUser(userId) if (keepActiveSession) { sessions = sessions.filter( @@ -190,10 +188,8 @@ exports.platformLogout = async ({ ctx, userId, keepActiveSession }) => { exports.clearCookie(ctx, Cookies.CurrentApp) } - await invalidateSessions( - userId, - sessions.map(({ sessionId }) => sessionId) - ) + const sessionIds = sessions.map(({ sessionId }) => sessionId) + await invalidateSessions(userId, { sessionIds, reason: "logout" }) await events.auth.logout() await userCache.invalidateUser(userId) } diff --git a/packages/backend-core/tests/utilities/mocks/events.js b/packages/backend-core/tests/utilities/mocks/events.js index a4055cc5e..415d59019 100644 --- a/packages/backend-core/tests/utilities/mocks/events.js +++ b/packages/backend-core/tests/utilities/mocks/events.js @@ -89,6 +89,14 @@ jest.spyOn(events.user, "passwordUpdated") jest.spyOn(events.user, "passwordResetRequested") jest.spyOn(events.user, "passwordReset") +jest.spyOn(events.group, "created") +jest.spyOn(events.group, "updated") +jest.spyOn(events.group, "deleted") +jest.spyOn(events.group, "usersAdded") +jest.spyOn(events.group, "usersDeleted") +jest.spyOn(events.group, "createdOnboarding") +jest.spyOn(events.group, "permissionsEdited") + jest.spyOn(events.serve, "servedBuilder") jest.spyOn(events.serve, "servedApp") jest.spyOn(events.serve, "servedAppPreview") diff --git a/packages/backend-core/tests/utilities/mocks/index.js b/packages/backend-core/tests/utilities/mocks/index.js index 3dd5c854c..6aa1c4a54 100644 --- a/packages/backend-core/tests/utilities/mocks/index.js +++ b/packages/backend-core/tests/utilities/mocks/index.js @@ -1,7 +1,9 @@ +const posthog = require("./posthog") const events = require("./events") const date = require("./date") module.exports = { + posthog, date, events, } diff --git a/packages/backend-core/tests/utilities/mocks/posthog.ts b/packages/backend-core/tests/utilities/mocks/posthog.ts new file mode 100644 index 000000000..e9cc653cc --- /dev/null +++ b/packages/backend-core/tests/utilities/mocks/posthog.ts @@ -0,0 +1,7 @@ +jest.mock("posthog-node", () => { + return jest.fn().mockImplementation(() => { + return { + capture: jest.fn(), + } + }) +}) diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock index e40cddc46..9f71691f4 100644 --- a/packages/backend-core/yarn.lock +++ b/packages/backend-core/yarn.lock @@ -291,6 +291,18 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== +"@hapi/hoek@^9.0.0": + version "9.3.0" + resolved "https://registry.yarnpkg.com/@hapi/hoek/-/hoek-9.3.0.tgz#8368869dcb735be2e7f5cb7647de78e167a251fb" + integrity sha512-/c6rf4UJlmHlC9b5BaNvzAcFv7HZ2QHaV0D4/HNlBdvFnvQq8RI4kYdhyPCl7Xj+oWvTWQ8ujhqS53LIgAe6KQ== + +"@hapi/topo@^5.0.0": + version "5.1.0" + resolved "https://registry.yarnpkg.com/@hapi/topo/-/topo-5.1.0.tgz#dc448e332c6c6e37a4dc02fd84ba8d44b9afb012" + integrity sha512-foQZKJig7Ob0BMAYBfcJk8d77QtOe7Wo4ox7ff1lQYoNNAb6jwcY1ncdoy2e9wQZzvNy7ODZCYJkK8kzmcAnAg== + dependencies: + "@hapi/hoek" "^9.0.0" + "@istanbuljs/load-nyc-config@^1.0.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz#fd3db1d59ecf7cf121e80650bb86712f9b55eced" @@ -539,6 +551,23 @@ koa "^2.13.4" node-mocks-http "^1.5.8" +"@sideway/address@^4.1.3": + version "4.1.4" + resolved "https://registry.yarnpkg.com/@sideway/address/-/address-4.1.4.tgz#03dccebc6ea47fdc226f7d3d1ad512955d4783f0" + integrity sha512-7vwq+rOHVWjyXxVlR76Agnvhy8I9rpzjosTESvmhNeXOXdZZB15Fl+TI9x1SiHZH5Jv2wTGduSxFDIaq0m3DUw== + dependencies: + "@hapi/hoek" "^9.0.0" + +"@sideway/formula@^3.0.0": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@sideway/formula/-/formula-3.0.0.tgz#fe158aee32e6bd5de85044be615bc08478a0a13c" + integrity sha512-vHe7wZ4NOXVfkoRb8T5otiENVlT7a3IAiw7H5M2+GO+9CDgcVUUsX1zalAztCmwyOr2RUTGJdgB+ZvSVqmdHmg== + +"@sideway/pinpoint@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@sideway/pinpoint/-/pinpoint-2.0.0.tgz#cff8ffadc372ad29fd3f78277aeb29e632cc70df" + integrity sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ== + "@sindresorhus/is@^0.14.0": version "0.14.0" resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.14.0.tgz#9fb3a3cf3132328151f353de4632e01e52102bea" @@ -764,6 +793,11 @@ "@types/koa-compose" "*" "@types/node" "*" +"@types/lodash@4.14.180": + version "4.14.180" + resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.180.tgz#4ab7c9ddfc92ec4a887886483bc14c79fb380670" + integrity sha512-XOKXa1KIxtNXgASAnwj7cnttJxS4fksBRywK/9LzRV5YxrF80BXZIGeQSuoESQ/VkUj30Ae0+YcuHc15wJCB2g== + "@types/mime@^1": version "1.3.2" resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a" @@ -3188,6 +3222,17 @@ jmespath@0.15.0: resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217" integrity sha512-+kHj8HXArPfpPEKGLZ+kB5ONRTCiGQXo8RQYL0hH8t6pWXUBBK5KkkQmTNOwKK4LEsd0yTsgtjJVm4UBSZea4w== +joi@17.6.0: + version "17.6.0" + resolved "https://registry.yarnpkg.com/joi/-/joi-17.6.0.tgz#0bb54f2f006c09a96e75ce687957bd04290054b2" + integrity sha512-OX5dG6DTbcr/kbMFj0KGYxuew69HPcAE3K/sZpEV2nP6e/j/C0HV+HNiBPCASxdx5T7DMoa0s8UeHWMnb6n2zw== + dependencies: + "@hapi/hoek" "^9.0.0" + "@hapi/topo" "^5.0.0" + "@sideway/address" "^4.1.3" + "@sideway/formula" "^3.0.0" + "@sideway/pinpoint" "^2.0.0" + join-component@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/join-component/-/join-component-1.1.0.tgz#b8417b750661a392bee2c2537c68b2a9d4977cd5" @@ -4123,6 +4168,11 @@ passport-oauth1@1.x.x: passport-strategy "1.x.x" utils-merge "1.x.x" +passport-oauth2-refresh@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/passport-oauth2-refresh/-/passport-oauth2-refresh-2.1.0.tgz#c31cd133826383f5539d16ad8ab4f35ca73ce4a4" + integrity sha512-4ML7ooCESCqiTgdDBzNUFTBcPR8zQq9iM6eppEUGMMvLdsjqRL93jKwWm4Az3OJcI+Q2eIVyI8sVRcPFvxcF/A== + passport-oauth2@1.x.x: version "1.6.1" resolved "https://registry.yarnpkg.com/passport-oauth2/-/passport-oauth2-1.6.1.tgz#c5aee8f849ce8bd436c7f81d904a3cd1666f181b" diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 6bb51dda7..cedd4722a 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "1.0.212-alpha.6", + "version": "1.2.39-alpha.5", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "^1.2.1", - "@budibase/string-templates": "^1.0.212-alpha.6", + "@budibase/string-templates": "1.2.39-alpha.5", "@spectrum-css/actionbutton": "^1.0.1", "@spectrum-css/actiongroup": "^1.0.1", "@spectrum-css/avatar": "^3.0.2", @@ -66,11 +66,12 @@ "@spectrum-css/radio": "^3.0.2", "@spectrum-css/search": "^3.0.2", "@spectrum-css/sidenav": "^3.0.2", + "@spectrum-css/slider": "3.0.1", "@spectrum-css/statuslight": "^3.0.2", "@spectrum-css/stepper": "^3.0.3", "@spectrum-css/switch": "^1.0.2", "@spectrum-css/table": "^3.0.1", - "@spectrum-css/tabs": "^3.0.1", + "@spectrum-css/tabs": "^3.2.12", "@spectrum-css/tags": "^3.0.2", "@spectrum-css/textfield": "^3.0.1", "@spectrum-css/toast": "^3.0.1", diff --git a/packages/bbui/src/ActionButton/ActionButton.svelte b/packages/bbui/src/ActionButton/ActionButton.svelte index b518ac3d9..cfc810807 100644 --- a/packages/bbui/src/ActionButton/ActionButton.svelte +++ b/packages/bbui/src/ActionButton/ActionButton.svelte @@ -13,6 +13,7 @@ export let size = "M" export let active = false export let fullWidth = false + export let noPadding = false function longPress(element) { if (!longPressable) return @@ -41,6 +42,7 @@ class:spectrum-ActionButton--quiet={quiet} class:spectrum-ActionButton--emphasized={emphasized} class:is-selected={selected} + class:noPadding class:fullWidth class="spectrum-ActionButton spectrum-ActionButton--size{size}" class:active @@ -80,4 +82,21 @@ .active svg { color: var(--spectrum-global-color-blue-600); } + :global([dir="ltr"] .spectrum-ActionButton .spectrum-Icon) { + margin-left: 0; + transition: color ease-out 130ms; + } + .is-selected:not(.spectrum-ActionButton--emphasized) { + background: var(--spectrum-global-color-gray-300); + } + .noPadding { + padding: 0; + min-width: 0; + } + .spectrum-ActionButton--quiet { + padding: 0 8px; + } + .is-selected:not(.emphasized) .spectrum-Icon { + color: var(--spectrum-global-color-gray-900); + } diff --git a/packages/bbui/src/Avatar/Avatar.svelte b/packages/bbui/src/Avatar/Avatar.svelte index f8acd9024..136a4fe24 100644 --- a/packages/bbui/src/Avatar/Avatar.svelte +++ b/packages/bbui/src/Avatar/Avatar.svelte @@ -4,7 +4,7 @@ ["XXS", "--spectrum-alias-avatar-size-50"], ["XS", "--spectrum-alias-avatar-size-75"], ["S", "--spectrum-alias-avatar-size-200"], - ["M", "--spectrum-alias-avatar-size-300"], + ["M", "--spectrum-alias-avatar-size-400"], ["L", "--spectrum-alias-avatar-size-500"], ["XL", "--spectrum-alias-avatar-size-600"], ["XXL", "--spectrum-alias-avatar-size-700"], @@ -13,6 +13,19 @@ export let url = "" export let disabled = false export let initials = "JD" + + const DefaultColor = "#3aab87" + + $: color = getColor(initials) + + const getColor = initials => { + if (!initials?.length) { + return DefaultColor + } + const code = initials[0].toLowerCase().charCodeAt(0) + const hue = ((code % 26) / 26) * 360 + return `hsl(${hue}, 50%, 50%)` + } {#if url} @@ -25,10 +38,11 @@ /> {:else}
{initials || ""}
@@ -40,7 +54,6 @@ display: grid; place-items: center; font-weight: 600; - background: #3aab87; border-radius: 50%; overflow: hidden; user-select: none; diff --git a/packages/bbui/src/Banner/Banner.svelte b/packages/bbui/src/Banner/Banner.svelte index f41fb5f80..3810021a6 100644 --- a/packages/bbui/src/Banner/Banner.svelte +++ b/packages/bbui/src/Banner/Banner.svelte @@ -8,6 +8,7 @@ export let size = "S" export let extraButtonText export let extraButtonAction + export let showCloseButton = true let show = true @@ -39,22 +40,24 @@ {/if} -
- -
+ {#if showCloseButton} +
+ +
+ {/if} {/if} @@ -63,4 +66,7 @@ pointer-events: all; width: 100%; } + .spectrum-Button { + border: 1px solid rgba(255, 255, 255, 0.2); + } diff --git a/packages/bbui/src/Divider/Divider.svelte b/packages/bbui/src/Divider/Divider.svelte index 2b4de9cfb..e4f0f2fb6 100644 --- a/packages/bbui/src/Divider/Divider.svelte +++ b/packages/bbui/src/Divider/Divider.svelte @@ -16,6 +16,9 @@ /> diff --git a/packages/bbui/src/Form/Core/Multiselect.svelte b/packages/bbui/src/Form/Core/Multiselect.svelte index 3eb1add26..9dd5a25a4 100644 --- a/packages/bbui/src/Form/Core/Multiselect.svelte +++ b/packages/bbui/src/Form/Core/Multiselect.svelte @@ -13,6 +13,7 @@ export let readonly = false export let autocomplete = false export let sort = false + export let autoWidth = false const dispatch = createEventDispatcher() $: selectedLookupMap = getSelectedLookupMap(value) @@ -85,4 +86,5 @@ {getOptionValue} onSelectOption={toggleOption} {sort} + {autoWidth} /> diff --git a/packages/bbui/src/Form/Core/Picker.svelte b/packages/bbui/src/Form/Core/Picker.svelte index cfd4b096b..cdaf00ade 100644 --- a/packages/bbui/src/Form/Core/Picker.svelte +++ b/packages/bbui/src/Form/Core/Picker.svelte @@ -6,12 +6,15 @@ import { createEventDispatcher } from "svelte" import clickOutside from "../../Actions/click_outside" import Search from "./Search.svelte" + import Icon from "../../Icon/Icon.svelte" + import StatusLight from "../../StatusLight/StatusLight.svelte" export let id = null export let disabled = false export let error = null export let fieldText = "" export let fieldIcon = "" + export let fieldColour = "" export let isPlaceholder = false export let placeholderOption = null export let options = [] @@ -20,6 +23,7 @@ export let getOptionLabel = option => option export let getOptionValue = option => option export let getOptionIcon = () => null + export let getOptionColour = () => null export let open = false export let readonly = false export let quiet = false @@ -83,11 +87,15 @@ on:mousedown={onClick} > {#if fieldIcon} - - icon + + + + {/if} + {#if fieldColour} + + {/if} - onSelectOption(getOptionValue(option, idx))} > {#if getOptionIcon(option, idx)} - - icon + + + + {/if} + {#if getOptionColour(option, idx)} + + {/if} @@ -201,6 +209,9 @@ width: 100%; box-shadow: none; } + .spectrum-Picker-label.auto-width { + margin-right: var(--spacing-xs); + } .spectrum-Picker-label:not(.auto-width) { overflow: hidden; text-overflow: ellipsis; @@ -213,14 +224,19 @@ .spectrum-Picker-label.auto-width.is-placeholder { padding-right: 2px; } + .auto-width .spectrum-Menu-item { + padding-right: var(--spacing-xl); + } - .icon-Padding { - padding-right: 10px; + /* Icon and colour alignment */ + .spectrum-Menu-checkmark { + align-self: center; + margin-top: 0; } - .icon-Placeholder-Padding { - padding-top: 5px; - padding-right: 10px; + .option-extra { + padding-right: 8px; } + .spectrum-Popover :global(.spectrum-Search) { margin-top: -1px; margin-left: -1px; diff --git a/packages/bbui/src/Form/Core/PickerDropdown.svelte b/packages/bbui/src/Form/Core/PickerDropdown.svelte new file mode 100644 index 000000000..28cb2b2a4 --- /dev/null +++ b/packages/bbui/src/Form/Core/PickerDropdown.svelte @@ -0,0 +1,436 @@ + + +
+
+ {#if iconData} + + + + {/if} + (primaryOpen = true)} + on:blur + on:focus + on:input + on:keyup + on:blur={onBlur} + on:input={onInput} + on:keyup={updateValueOnEnter} + value={primaryLabel || ""} + placeholder={placeholder || ""} + {disabled} + readonly + class="spectrum-Textfield-input spectrum-InputGroup-input" + class:labelPadding={iconData} + class:open={primaryOpen} + /> + {#if primaryValue && showClearIcon} + + {/if} +
+ {#if primaryOpen} +
(primaryOpen = false)} + transition:fly|local={{ y: -20, duration: 200 }} + class="spectrum-Popover spectrum-Popover--bottom spectrum-Picker-popover is-open" + class:auto-width={autoWidth} + class:is-full-width={!secondaryOptions.length} + > + {#if autocomplete} + updateSearch(event)} + {disabled} + placeholder="Search" + /> + {/if} + +
    + {#if placeholderOption} +
  • onSelectOption(null)} + > + {placeholderOption} + +
    • + {/if} + {#each groupTitles as title} +
    + {title} +
    + {#if primaryOptions} + {#each primaryOptions[title].data as option, idx} +
  • + onPickPrimary({ + value: primaryOptions[title].getValue(option), + label: primaryOptions[title].getLabel(option), + })} + > + {#if primaryOptions[title].getIcon(option)} +
    +
    + +
    +
    + {:else if getPrimaryOptionColour(option, idx)} + + + + {/if} + + + {primaryOptions[title].getLabel(option)} + + + + {#if getPrimaryOptionIcon(option, idx) && getPrimaryOptionColour(option, idx)} + + + + {/if} + +
    • + {/each} + {/if} + {/each} +
+
+ {/if} + {#if secondaryOptions.length} +
+ + {#if secondaryOpen} +
(secondaryOpen = false)} + transition:fly|local={{ y: -20, duration: 200 }} + class="spectrum-Popover spectrum-Popover--bottom spectrum-Picker-popover is-open" + style="width: 30%" + > +
    + {#each secondaryOptions as option, idx} +
  • + onPickSecondary(getSecondaryOptionValue(option, idx))} + > + {#if getSecondaryOptionColour(option, idx)} + + + + {/if} + + + {getSecondaryOptionLabel(option, idx)} + + +
    • + {/each} +
+
+ {/if} +
+ {/if} +
+ + diff --git a/packages/bbui/src/Form/Core/Search.svelte b/packages/bbui/src/Form/Core/Search.svelte index e6384e242..f19c2a7f4 100644 --- a/packages/bbui/src/Form/Core/Search.svelte +++ b/packages/bbui/src/Form/Core/Search.svelte @@ -8,6 +8,8 @@ export let id = null export let updateOnChange = true export let quiet = false + export let inputRef + const dispatch = createEventDispatcher() let focus = false @@ -68,6 +70,7 @@ type="search" class="spectrum-Textfield-input spectrum-Search-input" autocomplete="off" + bind:this={inputRef} /> + {/if} {#if showConfirmButton} diff --git a/packages/bbui/src/Notification/Notification.svelte b/packages/bbui/src/Notification/Notification.svelte index 1d2113155..53ab06270 100644 --- a/packages/bbui/src/Notification/Notification.svelte +++ b/packages/bbui/src/Notification/Notification.svelte @@ -1,15 +1,20 @@ -
+
{#if icon} {/if} -
+
{message || ""}
+ {#if action} + +
{actionMessage}
+ + {/if}
{#if dismissable}
@@ -46,4 +56,15 @@ .spectrum-Toast { pointer-events: all; } + + .wide { + width: 100%; + } + + .actionBody { + justify-content: space-between; + display: flex; + width: 100%; + align-items: center; + } diff --git a/packages/bbui/src/Notification/NotificationDisplay.svelte b/packages/bbui/src/Notification/NotificationDisplay.svelte index eb778f3aa..0f7e93eb2 100644 --- a/packages/bbui/src/Notification/NotificationDisplay.svelte +++ b/packages/bbui/src/Notification/NotificationDisplay.svelte @@ -8,13 +8,15 @@
- {#each $notifications as { type, icon, message, id, dismissable } (id)} -
+ {#each $notifications as { type, icon, message, id, dismissable, action, wide } (id)} +
notifications.dismiss(id)} />
@@ -25,7 +27,7 @@ diff --git a/packages/bbui/src/Stores/notifications.js b/packages/bbui/src/Stores/notifications.js index 74eed8628..449d282f2 100644 --- a/packages/bbui/src/Stores/notifications.js +++ b/packages/bbui/src/Stores/notifications.js @@ -20,7 +20,16 @@ export const createNotificationStore = () => { setTimeout(() => (block = false), timeout) } - const send = (message, type = "default", icon = "", autoDismiss = true) => { + const send = ( + message, + { + type = "default", + icon = "", + autoDismiss = true, + action = null, + wide = false, + } + ) => { if (block) { return } @@ -28,7 +37,15 @@ export const createNotificationStore = () => { _notifications.update(state => { return [ ...state, - { id: _id, type, message, icon, dismissable: !autoDismiss }, + { + id: _id, + type, + message, + icon, + dismissable: !autoDismiss, + action, + wide, + }, ] }) if (autoDismiss) { @@ -50,10 +67,11 @@ export const createNotificationStore = () => { return { subscribe, send, - info: msg => send(msg, "info", "Info"), - error: msg => send(msg, "error", "Alert", false), - warning: msg => send(msg, "warning", "Alert"), - success: msg => send(msg, "success", "CheckmarkCircle"), + info: msg => send(msg, { type: "info", icon: "Info" }), + error: msg => + send(msg, { type: "error", icon: "Alert", autoDismiss: false }), + warning: msg => send(msg, { type: "warning", icon: "Alert" }), + success: msg => send(msg, { type: "success", icon: "CheckmarkCircle" }), blockNotifications, dismiss: dismissNotification, } diff --git a/packages/bbui/src/Table/ArrayRenderer.svelte b/packages/bbui/src/Table/ArrayRenderer.svelte index 679973a03..375585066 100644 --- a/packages/bbui/src/Table/ArrayRenderer.svelte +++ b/packages/bbui/src/Table/ArrayRenderer.svelte @@ -5,7 +5,7 @@ const displayLimit = 5 - $: badges = value?.slice(0, displayLimit) ?? [] + $: badges = Array.isArray(value) ? value.slice(0, displayLimit) : [] $: leftover = (value?.length ?? 0) - badges.length diff --git a/packages/bbui/src/Table/AttachmentRenderer.svelte b/packages/bbui/src/Table/AttachmentRenderer.svelte index 97ce1394c..4dff22aef 100644 --- a/packages/bbui/src/Table/AttachmentRenderer.svelte +++ b/packages/bbui/src/Table/AttachmentRenderer.svelte @@ -1,5 +1,4 @@
@@ -60,16 +81,13 @@
- {#if showTestStatus && testResult && testResult[0]} + {#if showTestStatus && testResult}
{testResult[0].outputs?.success || isTrigger - ? "Success" - : "Error"}{status?.message}
{/if} @@ -79,7 +97,7 @@ onSelect(block) }} > - +
diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte index fecd0fcc7..9bbca1680 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte @@ -15,16 +15,20 @@ let trigger = {} let schemaProperties = {} - // clone the trigger so we're not mutating the reference - $: trigger = cloneDeep( - $automationStore.selectedAutomation.automation.definition.trigger - ) + $: { + // clone the trigger so we're not mutating the reference + trigger = cloneDeep( + $automationStore.selectedAutomation.automation.definition.trigger + ) + + // get the outputs so we can define the fields + let schema = Object.entries(trigger.schema?.outputs?.properties || {}) - // get the outputs so we can define the fields - $: schemaProperties = Object.entries(trigger?.schema?.outputs?.properties) + if (trigger?.event === "app:trigger") { + schema = [["fields", { customType: "fields" }]] + } - if (!$automationStore.selectedAutomation.automation.testData) { - $automationStore.selectedAutomation.automation.testData = {} + schemaProperties = schema } // check to see if there is existing test data in the store @@ -51,7 +55,7 @@ $automationStore.selectedAutomation?.automation, testData ) - $automationStore.selectedAutomation.automation.showTestPanel = true + $automationStore.showTestPanel = true } catch (error) { notifications.error("Error testing notification") } diff --git a/packages/builder/src/components/automation/AutomationBuilder/TestDisplay.svelte b/packages/builder/src/components/automation/AutomationBuilder/TestDisplay.svelte new file mode 100644 index 000000000..3f01dbeeb --- /dev/null +++ b/packages/builder/src/components/automation/AutomationBuilder/TestDisplay.svelte @@ -0,0 +1,132 @@ + + +
+ {#each blocks as block, idx} +
+ {#if block.stepId !== ActionStepID.LOOP} + + {#if showParameters && showParameters[block.id]} + + {#if filteredResults?.[idx]?.outputs.iterations} +
+ +
+ +
+
+ {/if} + +
+ + +