forked from tsai/budibase
Andrew Kingston
4 years ago
59 changed files with 1469 additions and 687 deletions
@ -0,0 +1,17 @@ |
|||
#!/bin/bash |
|||
|
|||
echo ${TARGETBUILD} > /buildtarget.txt |
|||
if [[ "${TARGETBUILD}" = "aas" ]]; then |
|||
# Azure AppService uses /home for persisent data & SSH on port 2222 |
|||
mkdir -p /home/budibase/{minio,couchdb} |
|||
mkdir -p /home/budibase/couchdb/data |
|||
chown -R couchdb:couchdb /home/budibase/couchdb/ |
|||
apt update |
|||
apt-get install -y openssh-server |
|||
sed -i 's#dir=/opt/couchdb/data/search#dir=/home/budibase/couchdb/data/search#' /opt/clouseau/clouseau.ini |
|||
sed -i 's#/minio/minio server /minio &#/minio/minio server /home/budibase/minio &#' /runner.sh |
|||
sed -i 's#database_dir = ./data#database_dir = /home/budibase/couchdb/data#' /opt/couchdb/etc/default.ini |
|||
sed -i 's#view_index_dir = ./data#view_index_dir = /home/budibase/couchdb/data#' /opt/couchdb/etc/default.ini |
|||
sed -i "s/#Port 22/Port 2222/" /etc/ssh/sshd_config |
|||
/etc/init.d/ssh restart |
|||
fi |
|||
@ -0,0 +1,94 @@ |
|||
server { |
|||
listen 80 default_server; |
|||
listen [::]:80 default_server; |
|||
server_name _; |
|||
|
|||
client_max_body_size 1000m; |
|||
ignore_invalid_headers off; |
|||
proxy_buffering off; |
|||
# port_in_redirect off; |
|||
|
|||
location ^~ /.well-known/acme-challenge/ { |
|||
default_type "text/plain"; |
|||
root /var/www/html; |
|||
break; |
|||
} |
|||
location = /.well-known/acme-challenge/ { |
|||
return 404; |
|||
} |
|||
|
|||
location /app { |
|||
proxy_pass http://127.0.0.1:4001; |
|||
} |
|||
|
|||
location = / { |
|||
proxy_pass http://127.0.0.1:4001; |
|||
} |
|||
|
|||
location ~ ^/(builder|app_) { |
|||
proxy_http_version 1.1; |
|||
proxy_set_header Connection $connection_upgrade; |
|||
proxy_set_header Upgrade $http_upgrade; |
|||
proxy_set_header X-Real-IP $remote_addr; |
|||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; |
|||
proxy_pass http://127.0.0.1:4001; |
|||
} |
|||
|
|||
location ~ ^/api/(system|admin|global)/ { |
|||
proxy_pass http://127.0.0.1:4002; |
|||
} |
|||
|
|||
location /worker/ { |
|||
proxy_pass http://127.0.0.1:4002; |
|||
rewrite ^/worker/(.*)$ /$1 break; |
|||
} |
|||
|
|||
location /api/ { |
|||
# calls to the API are rate limited with bursting |
|||
limit_req zone=ratelimit burst=20 nodelay; |
|||
|
|||
# 120s timeout on API requests |
|||
proxy_read_timeout 120s; |
|||
proxy_connect_timeout 120s; |
|||
proxy_send_timeout 120s; |
|||
|
|||
proxy_http_version 1.1; |
|||
proxy_set_header Connection $connection_upgrade; |
|||
proxy_set_header Upgrade $http_upgrade; |
|||
proxy_set_header X-Real-IP $remote_addr; |
|||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; |
|||
|
|||
proxy_pass http://127.0.0.1:4001; |
|||
} |
|||
|
|||
location /db/ { |
|||
proxy_pass http://127.0.0.1:5984; |
|||
rewrite ^/db/(.*)$ /$1 break; |
|||
} |
|||
|
|||
location / { |
|||
proxy_set_header X-Real-IP $remote_addr; |
|||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; |
|||
proxy_set_header X-Forwarded-Proto $scheme; |
|||
|
|||
proxy_connect_timeout 300; |
|||
proxy_http_version 1.1; |
|||
proxy_set_header Connection ""; |
|||
chunked_transfer_encoding off; |
|||
proxy_pass http://127.0.0.1:9000; |
|||
} |
|||
|
|||
client_header_timeout 60; |
|||
client_body_timeout 60; |
|||
keepalive_timeout 60; |
|||
|
|||
# gzip |
|||
gzip on; |
|||
gzip_vary on; |
|||
gzip_proxied any; |
|||
gzip_comp_level 6; |
|||
gzip_types text/plain text/css text/xml application/json application/javascript application/rss+xml application/atom+xml image/svg+xml; |
|||
|
|||
|
|||
|
|||
} |
|||
@ -1,4 +1,4 @@ |
|||
#!/bin/bash |
|||
id=$(docker run -t -d -p 10000:10000 budibase:latest) |
|||
id=$(docker run -t -d -p 80:80 budibase:latest) |
|||
docker exec -it $id bash |
|||
docker kill $id |
|||
|
|||
@ -1,5 +1,5 @@ |
|||
module.exports = { |
|||
Client: require("./src/redis"), |
|||
utils: require("./src/redis/utils"), |
|||
clients: require("./src/redis/authRedis"), |
|||
clients: require("./src/redis/init"), |
|||
} |
|||
|
|||
@ -0,0 +1,92 @@ |
|||
import { getTenantId } from "../../context" |
|||
import redis from "../../redis/init" |
|||
import RedisWrapper from "../../redis" |
|||
|
|||
function generateTenantKey(key: string) { |
|||
const tenantId = getTenantId() |
|||
return `${key}:${tenantId}` |
|||
} |
|||
|
|||
export = class BaseCache { |
|||
client: RedisWrapper | undefined |
|||
|
|||
constructor(client: RedisWrapper | undefined = undefined) { |
|||
this.client = client |
|||
} |
|||
|
|||
async getClient() { |
|||
return !this.client ? await redis.getCacheClient() : this.client |
|||
} |
|||
|
|||
async keys(pattern: string) { |
|||
const client = await this.getClient() |
|||
return client.keys(pattern) |
|||
} |
|||
|
|||
/** |
|||
* Read only from the cache. |
|||
*/ |
|||
async get(key: string, opts = { useTenancy: true }) { |
|||
key = opts.useTenancy ? generateTenantKey(key) : key |
|||
const client = await this.getClient() |
|||
return client.get(key) |
|||
} |
|||
|
|||
/** |
|||
* Write to the cache. |
|||
*/ |
|||
async store( |
|||
key: string, |
|||
value: any, |
|||
ttl: number | null = null, |
|||
opts = { useTenancy: true } |
|||
) { |
|||
key = opts.useTenancy ? generateTenantKey(key) : key |
|||
const client = await this.getClient() |
|||
await client.store(key, value, ttl) |
|||
} |
|||
|
|||
/** |
|||
* Remove from cache. |
|||
*/ |
|||
async delete(key: string, opts = { useTenancy: true }) { |
|||
key = opts.useTenancy ? generateTenantKey(key) : key |
|||
const client = await this.getClient() |
|||
return client.delete(key) |
|||
} |
|||
|
|||
/** |
|||
* Read from the cache. Write to the cache if not exists. |
|||
*/ |
|||
async withCache( |
|||
key: string, |
|||
ttl: number, |
|||
fetchFn: any, |
|||
opts = { useTenancy: true } |
|||
) { |
|||
const cachedValue = await this.get(key, opts) |
|||
if (cachedValue) { |
|||
return cachedValue |
|||
} |
|||
|
|||
try { |
|||
const fetchedValue = await fetchFn() |
|||
|
|||
await this.store(key, fetchedValue, ttl, opts) |
|||
return fetchedValue |
|||
} catch (err) { |
|||
console.error("Error fetching before cache - ", err) |
|||
throw err |
|||
} |
|||
} |
|||
|
|||
async bustCache(key: string, opts = { client: null }) { |
|||
const client = await this.getClient() |
|||
try { |
|||
await client.delete(generateTenantKey(key)) |
|||
} catch (err) { |
|||
console.error("Error busting cache - ", err) |
|||
throw err |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,59 @@ |
|||
require("../../../tests/utilities/TestConfiguration") |
|||
const { Writethrough } = require("../writethrough") |
|||
const { dangerousGetDB } = require("../../db") |
|||
const tk = require("timekeeper") |
|||
|
|||
const START_DATE = Date.now() |
|||
tk.freeze(START_DATE) |
|||
|
|||
const DELAY = 5000 |
|||
|
|||
const db = dangerousGetDB("test") |
|||
const db2 = dangerousGetDB("test2") |
|||
const writethrough = new Writethrough(db, DELAY), writethrough2 = new Writethrough(db2, DELAY) |
|||
|
|||
describe("writethrough", () => { |
|||
describe("put", () => { |
|||
let first |
|||
it("should be able to store, will go to DB", async () => { |
|||
const response = await writethrough.put({ _id: "test", value: 1 }) |
|||
const output = await db.get(response.id) |
|||
first = output |
|||
expect(output.value).toBe(1) |
|||
}) |
|||
|
|||
it("second put shouldn't update DB", async () => { |
|||
const response = await writethrough.put({ ...first, value: 2 }) |
|||
const output = await db.get(response.id) |
|||
expect(first._rev).toBe(output._rev) |
|||
expect(output.value).toBe(1) |
|||
}) |
|||
|
|||
it("should put it again after delay period", async () => { |
|||
tk.freeze(START_DATE + DELAY + 1) |
|||
const response = await writethrough.put({ ...first, value: 3 }) |
|||
const output = await db.get(response.id) |
|||
expect(response.rev).not.toBe(first._rev) |
|||
expect(output.value).toBe(3) |
|||
}) |
|||
}) |
|||
|
|||
describe("get", () => { |
|||
it("should be able to retrieve", async () => { |
|||
const response = await writethrough.get("test") |
|||
expect(response.value).toBe(3) |
|||
}) |
|||
}) |
|||
|
|||
describe("same doc, different databases (tenancy)", () => { |
|||
it("should be able to two different databases", async () => { |
|||
const resp1 = await writethrough.put({ _id: "db1", value: "first" }) |
|||
const resp2 = await writethrough2.put({ _id: "db1", value: "second" }) |
|||
expect(resp1.rev).toBeDefined() |
|||
expect(resp2.rev).toBeDefined() |
|||
expect((await db.get("db1")).value).toBe("first") |
|||
expect((await db2.get("db1")).value).toBe("second") |
|||
}) |
|||
}) |
|||
}) |
|||
|
|||
@ -0,0 +1,120 @@ |
|||
import BaseCache from "./base" |
|||
import { getWritethroughClient } from "../redis/init" |
|||
|
|||
const DEFAULT_WRITE_RATE_MS = 10000 |
|||
let CACHE: BaseCache | null = null |
|||
|
|||
interface CacheItem { |
|||
doc: any |
|||
lastWrite: number |
|||
} |
|||
|
|||
async function getCache() { |
|||
if (!CACHE) { |
|||
const client = await getWritethroughClient() |
|||
CACHE = new BaseCache(client) |
|||
} |
|||
return CACHE |
|||
} |
|||
|
|||
function makeCacheKey(db: PouchDB.Database, key: string) { |
|||
return db.name + key |
|||
} |
|||
|
|||
function makeCacheItem(doc: any, lastWrite: number | null = null): CacheItem { |
|||
return { doc, lastWrite: lastWrite || Date.now() } |
|||
} |
|||
|
|||
export async function put( |
|||
db: PouchDB.Database, |
|||
doc: any, |
|||
writeRateMs: number = DEFAULT_WRITE_RATE_MS |
|||
) { |
|||
const cache = await getCache() |
|||
const key = doc._id |
|||
let cacheItem: CacheItem | undefined = await cache.get(makeCacheKey(db, key)) |
|||
const updateDb = !cacheItem || cacheItem.lastWrite < Date.now() - writeRateMs |
|||
let output = doc |
|||
if (updateDb) { |
|||
const writeDb = async (toWrite: any) => { |
|||
// doc should contain the _id and _rev
|
|||
const response = await db.put(toWrite) |
|||
output = { |
|||
...doc, |
|||
_id: response.id, |
|||
_rev: response.rev, |
|||
} |
|||
} |
|||
try { |
|||
await writeDb(doc) |
|||
} catch (err: any) { |
|||
if (err.status !== 409) { |
|||
throw err |
|||
} else { |
|||
// get the rev, update over it - this is risky, may change in future
|
|||
const readDoc = await db.get(doc._id) |
|||
doc._rev = readDoc._rev |
|||
await writeDb(doc) |
|||
} |
|||
} |
|||
} |
|||
// if we are updating the DB then need to set the lastWrite to now
|
|||
cacheItem = makeCacheItem(output, updateDb ? null : cacheItem?.lastWrite) |
|||
await cache.store(makeCacheKey(db, key), cacheItem) |
|||
return { ok: true, id: output._id, rev: output._rev } |
|||
} |
|||
|
|||
export async function get(db: PouchDB.Database, id: string): Promise<any> { |
|||
const cache = await getCache() |
|||
const cacheKey = makeCacheKey(db, id) |
|||
let cacheItem: CacheItem = await cache.get(cacheKey) |
|||
if (!cacheItem) { |
|||
const doc = await db.get(id) |
|||
cacheItem = makeCacheItem(doc) |
|||
await cache.store(cacheKey, cacheItem) |
|||
} |
|||
return cacheItem.doc |
|||
} |
|||
|
|||
export async function remove( |
|||
db: PouchDB.Database, |
|||
docOrId: any, |
|||
rev?: any |
|||
): Promise<void> { |
|||
const cache = await getCache() |
|||
if (!docOrId) { |
|||
throw new Error("No ID/Rev provided.") |
|||
} |
|||
const id = typeof docOrId === "string" ? docOrId : docOrId._id |
|||
rev = typeof docOrId === "string" ? rev : docOrId._rev |
|||
try { |
|||
await cache.delete(makeCacheKey(db, id)) |
|||
} finally { |
|||
await db.remove(id, rev) |
|||
} |
|||
} |
|||
|
|||
export class Writethrough { |
|||
db: PouchDB.Database |
|||
writeRateMs: number |
|||
|
|||
constructor( |
|||
db: PouchDB.Database, |
|||
writeRateMs: number = DEFAULT_WRITE_RATE_MS |
|||
) { |
|||
this.db = db |
|||
this.writeRateMs = writeRateMs |
|||
} |
|||
|
|||
async put(doc: any) { |
|||
return put(this.db, doc, this.writeRateMs) |
|||
} |
|||
|
|||
async get(id: string) { |
|||
return get(this.db, id) |
|||
} |
|||
|
|||
async remove(docOrId: any, rev?: any) { |
|||
return remove(this.db, docOrId, rev) |
|||
} |
|||
} |
|||
@ -0,0 +1,7 @@ |
|||
node_modules |
|||
npm-debug.log |
|||
Dockerfile |
|||
.dockerignore |
|||
.git |
|||
.gitignore |
|||
|
|||
File diff suppressed because it is too large
Loading…
Reference in new issue