mirror of https://github.com/Budibase/budibase.git
Andrew Kingston
5 years ago
83 changed files with 10141 additions and 8010 deletions
@ -1,3 +1,4 @@ |
|||
module.exports = { |
|||
user: require("./src/cache/user"), |
|||
app: require("./src/cache/appMetadata"), |
|||
} |
|||
|
|||
@ -0,0 +1,85 @@ |
|||
const redis = require("../redis/authRedis") |
|||
const { getCouch } = require("../db") |
|||
const { DocumentTypes } = require("../db/constants") |
|||
|
|||
const AppState = { |
|||
INVALID: "invalid", |
|||
} |
|||
const EXPIRY_SECONDS = 3600 |
|||
|
|||
/** |
|||
* The default populate app metadata function |
|||
*/ |
|||
const populateFromDB = async (appId, CouchDB = null) => { |
|||
if (!CouchDB) { |
|||
CouchDB = getCouch() |
|||
} |
|||
const db = new CouchDB(appId, { skip_setup: true }) |
|||
return db.get(DocumentTypes.APP_METADATA) |
|||
} |
|||
|
|||
const isInvalid = metadata => { |
|||
return !metadata || metadata.state === AppState.INVALID |
|||
} |
|||
|
|||
/** |
|||
* Get the requested app metadata by id. |
|||
* Use redis cache to first read the app metadata. |
|||
* If not present fallback to loading the app metadata directly and re-caching. |
|||
* @param {string} appId the id of the app to get metadata from. |
|||
* @param {object} CouchDB the database being passed |
|||
* @returns {object} the app metadata. |
|||
*/ |
|||
exports.getAppMetadata = async (appId, CouchDB = null) => { |
|||
const client = await redis.getAppClient() |
|||
// try cache
|
|||
let metadata = await client.get(appId) |
|||
if (!metadata) { |
|||
let expiry = EXPIRY_SECONDS |
|||
try { |
|||
metadata = await populateFromDB(appId, CouchDB) |
|||
} catch (err) { |
|||
// app DB left around, but no metadata, it is invalid
|
|||
if (err && err.status === 404) { |
|||
metadata = { state: AppState.INVALID } |
|||
// don't expire the reference to an invalid app, it'll only be
|
|||
// updated if a metadata doc actually gets stored (app is remade/reverted)
|
|||
expiry = undefined |
|||
} else { |
|||
throw err |
|||
} |
|||
} |
|||
// needed for cypress/some scenarios where the caching happens
|
|||
// so quickly the requests can get slightly out of sync
|
|||
// might store its invalid just before it stores its valid
|
|||
if (isInvalid(metadata)) { |
|||
const temp = await client.get(appId) |
|||
if (temp) { |
|||
metadata = temp |
|||
} |
|||
} |
|||
await client.store(appId, metadata, expiry) |
|||
} |
|||
// we've stored in the cache an object to tell us that it is currently invalid
|
|||
if (isInvalid(metadata)) { |
|||
throw { status: 404, message: "No app metadata found" } |
|||
} |
|||
return metadata |
|||
} |
|||
|
|||
/** |
|||
* Invalidate/reset the cached metadata when a change occurs in the db. |
|||
* @param appId {string} the cache key to bust/update. |
|||
* @param newMetadata {object|undefined} optional - can simply provide the new metadata to update with. |
|||
* @return {Promise<void>} will respond with success when cache is updated. |
|||
*/ |
|||
exports.invalidateAppMetadata = async (appId, newMetadata = null) => { |
|||
if (!appId) { |
|||
throw "Cannot invalidate if no app ID provided." |
|||
} |
|||
const client = await redis.getAppClient() |
|||
await client.delete(appId) |
|||
if (newMetadata) { |
|||
await client.store(appId, newMetadata, EXPIRY_SECONDS) |
|||
} |
|||
} |
|||
File diff suppressed because it is too large
@ -1,16 +1,67 @@ |
|||
<script> |
|||
import "@spectrum-css/fieldlabel/dist/index-vars.css" |
|||
import Tooltip from "../Tooltip/Tooltip.svelte" |
|||
import Icon from "../Icon/Icon.svelte" |
|||
|
|||
export let size = "M" |
|||
export let tooltip = "" |
|||
export let showTooltip = false |
|||
</script> |
|||
|
|||
<label for="" class={`spectrum-FieldLabel spectrum-FieldLabel--size${size}`}> |
|||
<slot /> |
|||
</label> |
|||
{#if tooltip} |
|||
<div class="container"> |
|||
<label |
|||
for="" |
|||
class={`spectrum-FieldLabel spectrum-FieldLabel--size${size}`} |
|||
> |
|||
<slot /> |
|||
</label> |
|||
<div class="icon-container"> |
|||
<div |
|||
class="icon" |
|||
on:mouseover={() => (showTooltip = true)} |
|||
on:mouseleave={() => (showTooltip = false)} |
|||
> |
|||
<Icon name="InfoOutline" size="S" disabled={true} /> |
|||
</div> |
|||
{#if showTooltip} |
|||
<div class="tooltip"> |
|||
<Tooltip textWrapping={true} direction={"bottom"} text={tooltip} /> |
|||
</div> |
|||
{/if} |
|||
</div> |
|||
</div> |
|||
{:else} |
|||
<label for="" class={`spectrum-FieldLabel spectrum-FieldLabel--size${size}`}> |
|||
<slot /> |
|||
</label> |
|||
{/if} |
|||
|
|||
<style> |
|||
label { |
|||
padding: 0; |
|||
white-space: nowrap; |
|||
} |
|||
.container { |
|||
display: flex; |
|||
} |
|||
.icon-container { |
|||
position: relative; |
|||
display: flex; |
|||
justify-content: center; |
|||
margin-top: 1px; |
|||
margin-left: 5px; |
|||
margin-right: 5px; |
|||
} |
|||
.tooltip { |
|||
position: absolute; |
|||
display: flex; |
|||
justify-content: center; |
|||
top: 15px; |
|||
z-index: 1; |
|||
width: 160px; |
|||
} |
|||
.icon { |
|||
transform: scale(0.75); |
|||
} |
|||
</style> |
|||
|
|||
File diff suppressed because it is too large
File diff suppressed because it is too large
File diff suppressed because it is too large
@ -0,0 +1,33 @@ |
|||
/****************************************************** |
|||
* This script just makes it easy to re-create * |
|||
* a cypress like environment for testing the backend * |
|||
******************************************************/ |
|||
const path = require("path") |
|||
const tmpdir = path.join(require("os").tmpdir(), ".budibase") |
|||
|
|||
const MAIN_PORT = "10001" |
|||
const WORKER_PORT = "10002" |
|||
|
|||
// @ts-ignore
|
|||
process.env.PORT = MAIN_PORT |
|||
process.env.BUDIBASE_API_KEY = "6BE826CB-6B30-4AEC-8777-2E90464633DE" |
|||
process.env.NODE_ENV = "cypress" |
|||
process.env.ENABLE_ANALYTICS = "false" |
|||
process.env.JWT_SECRET = "budibase" |
|||
process.env.COUCH_URL = `leveldb://${tmpdir}/.data/` |
|||
process.env.SELF_HOSTED = "1" |
|||
process.env.WORKER_URL = `http://localhost:${WORKER_PORT}/` |
|||
process.env.MINIO_URL = `http://localhost:${MAIN_PORT}/` |
|||
process.env.MINIO_ACCESS_KEY = "budibase" |
|||
process.env.MINIO_SECRET_KEY = "budibase" |
|||
process.env.COUCH_DB_USER = "budibase" |
|||
process.env.COUCH_DB_PASSWORD = "budibase" |
|||
process.env.INTERNAL_API_KEY = "budibase" |
|||
process.env.ALLOW_DEV_AUTOMATIONS = "1" |
|||
|
|||
// don't make this a variable or top level require
|
|||
// it will cause environment module to be loaded prematurely
|
|||
const server = require("../src/app") |
|||
process.env.PORT = WORKER_PORT |
|||
const worker = require("../../worker/src/index") |
|||
process.env.PORT = MAIN_PORT |
|||
@ -0,0 +1,6 @@ |
|||
export interface IntegrationBase { |
|||
create?(query: any): Promise<[any]> |
|||
read?(query: any): Promise<[any]> |
|||
update?(query: any): Promise<[any]> |
|||
delete?(query: any): Promise<[any]> |
|||
} |
|||
@ -0,0 +1,60 @@ |
|||
const workerFarm = require("worker-farm") |
|||
const env = require("../environment") |
|||
|
|||
const ThreadType = { |
|||
QUERY: "query", |
|||
AUTOMATION: "automation", |
|||
} |
|||
|
|||
function typeToFile(type) { |
|||
let filename = null |
|||
switch (type) { |
|||
case ThreadType.QUERY: |
|||
filename = "./query" |
|||
break |
|||
case ThreadType.AUTOMATION: |
|||
filename = "./automation" |
|||
break |
|||
default: |
|||
throw "Unknown thread type" |
|||
} |
|||
return require.resolve(filename) |
|||
} |
|||
|
|||
class Thread { |
|||
constructor(type, opts = { timeoutMs: null, count: 1 }) { |
|||
this.type = type |
|||
if (!env.isTest()) { |
|||
const workerOpts = { |
|||
autoStart: true, |
|||
maxConcurrentWorkers: opts.count ? opts.count : 1, |
|||
} |
|||
if (opts.timeoutMs) { |
|||
workerOpts.maxCallTime = opts.timeoutMs |
|||
} |
|||
this.workers = workerFarm(workerOpts, typeToFile(type)) |
|||
} |
|||
} |
|||
|
|||
run(data) { |
|||
return new Promise((resolve, reject) => { |
|||
let fncToCall |
|||
// if in test then don't use threading
|
|||
if (env.isTest()) { |
|||
fncToCall = require(typeToFile(this.type)) |
|||
} else { |
|||
fncToCall = this.workers |
|||
} |
|||
fncToCall(data, (err, response) => { |
|||
if (err) { |
|||
reject(err) |
|||
} else { |
|||
resolve(response) |
|||
} |
|||
}) |
|||
}) |
|||
} |
|||
} |
|||
|
|||
module.exports.Thread = Thread |
|||
module.exports.ThreadType = ThreadType |
|||
@ -0,0 +1,63 @@ |
|||
const ScriptRunner = require("../utilities/scriptRunner") |
|||
const { integrations } = require("../integrations") |
|||
|
|||
function formatResponse(resp) { |
|||
if (typeof resp === "string") { |
|||
try { |
|||
resp = JSON.parse(resp) |
|||
} catch (err) { |
|||
resp = { response: resp } |
|||
} |
|||
} |
|||
return resp |
|||
} |
|||
|
|||
async function runAndTransform(datasource, queryVerb, query, transformer) { |
|||
const Integration = integrations[datasource.source] |
|||
if (!Integration) { |
|||
throw "Integration type does not exist." |
|||
} |
|||
const integration = new Integration(datasource.config) |
|||
|
|||
let rows = formatResponse(await integration[queryVerb](query)) |
|||
|
|||
// transform as required
|
|||
if (transformer) { |
|||
const runner = new ScriptRunner(transformer, { data: rows }) |
|||
rows = runner.execute() |
|||
} |
|||
|
|||
// needs to an array for next step
|
|||
if (!Array.isArray(rows)) { |
|||
rows = [rows] |
|||
} |
|||
|
|||
// map into JSON if just raw primitive here
|
|||
if (rows.find(row => typeof row !== "object")) { |
|||
rows = rows.map(value => ({ value })) |
|||
} |
|||
|
|||
// get all the potential fields in the schema
|
|||
let keys = rows.flatMap(Object.keys) |
|||
|
|||
if (integration.end) { |
|||
integration.end() |
|||
} |
|||
|
|||
return { rows, keys } |
|||
} |
|||
|
|||
module.exports = (input, callback) => { |
|||
runAndTransform( |
|||
input.datasource, |
|||
input.queryVerb, |
|||
input.query, |
|||
input.transformer |
|||
) |
|||
.then(response => { |
|||
callback(null, response) |
|||
}) |
|||
.catch(err => { |
|||
callback(err) |
|||
}) |
|||
} |
|||
File diff suppressed because it is too large
File diff suppressed because it is too large
File diff suppressed because it is too large
@ -0,0 +1,54 @@ |
|||
const fs = require("fs") |
|||
const { join } = require("path") |
|||
const { spawnSync } =require("child_process") |
|||
|
|||
const DONT_RUN_PKG = ["bbui"] |
|||
const PACKAGES_PATH = join(__dirname, "..", "packages") |
|||
|
|||
function getPackages() { |
|||
return fs.readdirSync(PACKAGES_PATH) |
|||
} |
|||
|
|||
function deleteFile(path) { |
|||
try { |
|||
fs.unlinkSync(path) |
|||
} catch (err) { |
|||
// don't error, it just doesn't exist
|
|||
} |
|||
} |
|||
|
|||
function removeModules(path) { |
|||
if (fs.existsSync(path)) { |
|||
fs.rmdirSync(path, { recursive: true }) |
|||
} |
|||
} |
|||
|
|||
function executeInPackage(packageName) { |
|||
if (DONT_RUN_PKG.includes(packageName)) { |
|||
return |
|||
} |
|||
const dir = join(PACKAGES_PATH, packageName) |
|||
if (!fs.existsSync(join(dir, "package.json"))) { |
|||
console.error(`SKIPPING ${packageName} directory, no package.json`) |
|||
return |
|||
} |
|||
const packageLockLoc = join(dir, "package-lock.json") |
|||
const modulesLoc = join(dir, "node_modules") |
|||
deleteFile(join(dir, "yarn.lock")) |
|||
deleteFile(packageLockLoc) |
|||
removeModules(modulesLoc) |
|||
const opts = { cwd: dir, stdio: "inherit", shell: true } |
|||
spawnSync("npm", ["i", "--package-lock-only"], opts) |
|||
spawnSync("npm", ["audit", "fix"], opts) |
|||
spawnSync("yarn", ["import"], opts) |
|||
deleteFile(packageLockLoc) |
|||
removeModules(modulesLoc) |
|||
} |
|||
|
|||
const packages = getPackages() |
|||
for (let pkg of packages) { |
|||
executeInPackage(pkg) |
|||
} |
|||
|
|||
spawnSync("yarn", ["bootstrap"], { cwd: join(__dirname, ".."), stdio: "inherit", shell: true }) |
|||
|
|||
File diff suppressed because it is too large
Loading…
Reference in new issue