mirror of https://github.com/Budibase/budibase.git
Andrew Kingston
5 years ago
83 changed files with 10141 additions and 8010 deletions
@ -1,3 +1,4 @@ |
|||||
module.exports = { |
module.exports = { |
||||
user: require("./src/cache/user"), |
user: require("./src/cache/user"), |
||||
|
app: require("./src/cache/appMetadata"), |
||||
} |
} |
||||
|
|||||
@ -0,0 +1,85 @@ |
|||||
|
const redis = require("../redis/authRedis") |
||||
|
const { getCouch } = require("../db") |
||||
|
const { DocumentTypes } = require("../db/constants") |
||||
|
|
||||
|
const AppState = { |
||||
|
INVALID: "invalid", |
||||
|
} |
||||
|
const EXPIRY_SECONDS = 3600 |
||||
|
|
||||
|
/** |
||||
|
* The default populate app metadata function |
||||
|
*/ |
||||
|
const populateFromDB = async (appId, CouchDB = null) => { |
||||
|
if (!CouchDB) { |
||||
|
CouchDB = getCouch() |
||||
|
} |
||||
|
const db = new CouchDB(appId, { skip_setup: true }) |
||||
|
return db.get(DocumentTypes.APP_METADATA) |
||||
|
} |
||||
|
|
||||
|
const isInvalid = metadata => { |
||||
|
return !metadata || metadata.state === AppState.INVALID |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* Get the requested app metadata by id. |
||||
|
* Use redis cache to first read the app metadata. |
||||
|
* If not present fallback to loading the app metadata directly and re-caching. |
||||
|
* @param {string} appId the id of the app to get metadata from. |
||||
|
* @param {object} CouchDB the database being passed |
||||
|
* @returns {object} the app metadata. |
||||
|
*/ |
||||
|
exports.getAppMetadata = async (appId, CouchDB = null) => { |
||||
|
const client = await redis.getAppClient() |
||||
|
// try cache
|
||||
|
let metadata = await client.get(appId) |
||||
|
if (!metadata) { |
||||
|
let expiry = EXPIRY_SECONDS |
||||
|
try { |
||||
|
metadata = await populateFromDB(appId, CouchDB) |
||||
|
} catch (err) { |
||||
|
// app DB left around, but no metadata, it is invalid
|
||||
|
if (err && err.status === 404) { |
||||
|
metadata = { state: AppState.INVALID } |
||||
|
// don't expire the reference to an invalid app, it'll only be
|
||||
|
// updated if a metadata doc actually gets stored (app is remade/reverted)
|
||||
|
expiry = undefined |
||||
|
} else { |
||||
|
throw err |
||||
|
} |
||||
|
} |
||||
|
// needed for cypress/some scenarios where the caching happens
|
||||
|
// so quickly the requests can get slightly out of sync
|
||||
|
// might store its invalid just before it stores its valid
|
||||
|
if (isInvalid(metadata)) { |
||||
|
const temp = await client.get(appId) |
||||
|
if (temp) { |
||||
|
metadata = temp |
||||
|
} |
||||
|
} |
||||
|
await client.store(appId, metadata, expiry) |
||||
|
} |
||||
|
// we've stored in the cache an object to tell us that it is currently invalid
|
||||
|
if (isInvalid(metadata)) { |
||||
|
throw { status: 404, message: "No app metadata found" } |
||||
|
} |
||||
|
return metadata |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* Invalidate/reset the cached metadata when a change occurs in the db. |
||||
|
* @param appId {string} the cache key to bust/update. |
||||
|
* @param newMetadata {object|undefined} optional - can simply provide the new metadata to update with. |
||||
|
* @return {Promise<void>} will respond with success when cache is updated. |
||||
|
*/ |
||||
|
exports.invalidateAppMetadata = async (appId, newMetadata = null) => { |
||||
|
if (!appId) { |
||||
|
throw "Cannot invalidate if no app ID provided." |
||||
|
} |
||||
|
const client = await redis.getAppClient() |
||||
|
await client.delete(appId) |
||||
|
if (newMetadata) { |
||||
|
await client.store(appId, newMetadata, EXPIRY_SECONDS) |
||||
|
} |
||||
|
} |
||||
File diff suppressed because it is too large
@ -1,16 +1,67 @@ |
|||||
<script> |
<script> |
||||
import "@spectrum-css/fieldlabel/dist/index-vars.css" |
import "@spectrum-css/fieldlabel/dist/index-vars.css" |
||||
|
import Tooltip from "../Tooltip/Tooltip.svelte" |
||||
|
import Icon from "../Icon/Icon.svelte" |
||||
|
|
||||
export let size = "M" |
export let size = "M" |
||||
|
export let tooltip = "" |
||||
|
export let showTooltip = false |
||||
</script> |
</script> |
||||
|
|
||||
<label for="" class={`spectrum-FieldLabel spectrum-FieldLabel--size${size}`}> |
{#if tooltip} |
||||
<slot /> |
<div class="container"> |
||||
</label> |
<label |
||||
|
for="" |
||||
|
class={`spectrum-FieldLabel spectrum-FieldLabel--size${size}`} |
||||
|
> |
||||
|
<slot /> |
||||
|
</label> |
||||
|
<div class="icon-container"> |
||||
|
<div |
||||
|
class="icon" |
||||
|
on:mouseover={() => (showTooltip = true)} |
||||
|
on:mouseleave={() => (showTooltip = false)} |
||||
|
> |
||||
|
<Icon name="InfoOutline" size="S" disabled={true} /> |
||||
|
</div> |
||||
|
{#if showTooltip} |
||||
|
<div class="tooltip"> |
||||
|
<Tooltip textWrapping={true} direction={"bottom"} text={tooltip} /> |
||||
|
</div> |
||||
|
{/if} |
||||
|
</div> |
||||
|
</div> |
||||
|
{:else} |
||||
|
<label for="" class={`spectrum-FieldLabel spectrum-FieldLabel--size${size}`}> |
||||
|
<slot /> |
||||
|
</label> |
||||
|
{/if} |
||||
|
|
||||
<style> |
<style> |
||||
label { |
label { |
||||
padding: 0; |
padding: 0; |
||||
white-space: nowrap; |
white-space: nowrap; |
||||
} |
} |
||||
|
.container { |
||||
|
display: flex; |
||||
|
} |
||||
|
.icon-container { |
||||
|
position: relative; |
||||
|
display: flex; |
||||
|
justify-content: center; |
||||
|
margin-top: 1px; |
||||
|
margin-left: 5px; |
||||
|
margin-right: 5px; |
||||
|
} |
||||
|
.tooltip { |
||||
|
position: absolute; |
||||
|
display: flex; |
||||
|
justify-content: center; |
||||
|
top: 15px; |
||||
|
z-index: 1; |
||||
|
width: 160px; |
||||
|
} |
||||
|
.icon { |
||||
|
transform: scale(0.75); |
||||
|
} |
||||
</style> |
</style> |
||||
|
|||||
File diff suppressed because it is too large
File diff suppressed because it is too large
File diff suppressed because it is too large
@ -0,0 +1,33 @@ |
|||||
|
/****************************************************** |
||||
|
* This script just makes it easy to re-create * |
||||
|
* a cypress like environment for testing the backend * |
||||
|
******************************************************/ |
||||
|
const path = require("path") |
||||
|
const tmpdir = path.join(require("os").tmpdir(), ".budibase") |
||||
|
|
||||
|
const MAIN_PORT = "10001" |
||||
|
const WORKER_PORT = "10002" |
||||
|
|
||||
|
// @ts-ignore
|
||||
|
process.env.PORT = MAIN_PORT |
||||
|
process.env.BUDIBASE_API_KEY = "6BE826CB-6B30-4AEC-8777-2E90464633DE" |
||||
|
process.env.NODE_ENV = "cypress" |
||||
|
process.env.ENABLE_ANALYTICS = "false" |
||||
|
process.env.JWT_SECRET = "budibase" |
||||
|
process.env.COUCH_URL = `leveldb://${tmpdir}/.data/` |
||||
|
process.env.SELF_HOSTED = "1" |
||||
|
process.env.WORKER_URL = `http://localhost:${WORKER_PORT}/` |
||||
|
process.env.MINIO_URL = `http://localhost:${MAIN_PORT}/` |
||||
|
process.env.MINIO_ACCESS_KEY = "budibase" |
||||
|
process.env.MINIO_SECRET_KEY = "budibase" |
||||
|
process.env.COUCH_DB_USER = "budibase" |
||||
|
process.env.COUCH_DB_PASSWORD = "budibase" |
||||
|
process.env.INTERNAL_API_KEY = "budibase" |
||||
|
process.env.ALLOW_DEV_AUTOMATIONS = "1" |
||||
|
|
||||
|
// don't make this a variable or top level require
|
||||
|
// it will cause environment module to be loaded prematurely
|
||||
|
const server = require("../src/app") |
||||
|
process.env.PORT = WORKER_PORT |
||||
|
const worker = require("../../worker/src/index") |
||||
|
process.env.PORT = MAIN_PORT |
||||
@ -0,0 +1,6 @@ |
|||||
|
export interface IntegrationBase { |
||||
|
create?(query: any): Promise<[any]> |
||||
|
read?(query: any): Promise<[any]> |
||||
|
update?(query: any): Promise<[any]> |
||||
|
delete?(query: any): Promise<[any]> |
||||
|
} |
||||
@ -0,0 +1,60 @@ |
|||||
|
const workerFarm = require("worker-farm") |
||||
|
const env = require("../environment") |
||||
|
|
||||
|
const ThreadType = { |
||||
|
QUERY: "query", |
||||
|
AUTOMATION: "automation", |
||||
|
} |
||||
|
|
||||
|
function typeToFile(type) { |
||||
|
let filename = null |
||||
|
switch (type) { |
||||
|
case ThreadType.QUERY: |
||||
|
filename = "./query" |
||||
|
break |
||||
|
case ThreadType.AUTOMATION: |
||||
|
filename = "./automation" |
||||
|
break |
||||
|
default: |
||||
|
throw "Unknown thread type" |
||||
|
} |
||||
|
return require.resolve(filename) |
||||
|
} |
||||
|
|
||||
|
class Thread { |
||||
|
constructor(type, opts = { timeoutMs: null, count: 1 }) { |
||||
|
this.type = type |
||||
|
if (!env.isTest()) { |
||||
|
const workerOpts = { |
||||
|
autoStart: true, |
||||
|
maxConcurrentWorkers: opts.count ? opts.count : 1, |
||||
|
} |
||||
|
if (opts.timeoutMs) { |
||||
|
workerOpts.maxCallTime = opts.timeoutMs |
||||
|
} |
||||
|
this.workers = workerFarm(workerOpts, typeToFile(type)) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
run(data) { |
||||
|
return new Promise((resolve, reject) => { |
||||
|
let fncToCall |
||||
|
// if in test then don't use threading
|
||||
|
if (env.isTest()) { |
||||
|
fncToCall = require(typeToFile(this.type)) |
||||
|
} else { |
||||
|
fncToCall = this.workers |
||||
|
} |
||||
|
fncToCall(data, (err, response) => { |
||||
|
if (err) { |
||||
|
reject(err) |
||||
|
} else { |
||||
|
resolve(response) |
||||
|
} |
||||
|
}) |
||||
|
}) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
module.exports.Thread = Thread |
||||
|
module.exports.ThreadType = ThreadType |
||||
@ -0,0 +1,63 @@ |
|||||
|
const ScriptRunner = require("../utilities/scriptRunner") |
||||
|
const { integrations } = require("../integrations") |
||||
|
|
||||
|
function formatResponse(resp) { |
||||
|
if (typeof resp === "string") { |
||||
|
try { |
||||
|
resp = JSON.parse(resp) |
||||
|
} catch (err) { |
||||
|
resp = { response: resp } |
||||
|
} |
||||
|
} |
||||
|
return resp |
||||
|
} |
||||
|
|
||||
|
async function runAndTransform(datasource, queryVerb, query, transformer) { |
||||
|
const Integration = integrations[datasource.source] |
||||
|
if (!Integration) { |
||||
|
throw "Integration type does not exist." |
||||
|
} |
||||
|
const integration = new Integration(datasource.config) |
||||
|
|
||||
|
let rows = formatResponse(await integration[queryVerb](query)) |
||||
|
|
||||
|
// transform as required
|
||||
|
if (transformer) { |
||||
|
const runner = new ScriptRunner(transformer, { data: rows }) |
||||
|
rows = runner.execute() |
||||
|
} |
||||
|
|
||||
|
// needs to an array for next step
|
||||
|
if (!Array.isArray(rows)) { |
||||
|
rows = [rows] |
||||
|
} |
||||
|
|
||||
|
// map into JSON if just raw primitive here
|
||||
|
if (rows.find(row => typeof row !== "object")) { |
||||
|
rows = rows.map(value => ({ value })) |
||||
|
} |
||||
|
|
||||
|
// get all the potential fields in the schema
|
||||
|
let keys = rows.flatMap(Object.keys) |
||||
|
|
||||
|
if (integration.end) { |
||||
|
integration.end() |
||||
|
} |
||||
|
|
||||
|
return { rows, keys } |
||||
|
} |
||||
|
|
||||
|
module.exports = (input, callback) => { |
||||
|
runAndTransform( |
||||
|
input.datasource, |
||||
|
input.queryVerb, |
||||
|
input.query, |
||||
|
input.transformer |
||||
|
) |
||||
|
.then(response => { |
||||
|
callback(null, response) |
||||
|
}) |
||||
|
.catch(err => { |
||||
|
callback(err) |
||||
|
}) |
||||
|
} |
||||
File diff suppressed because it is too large
File diff suppressed because it is too large
File diff suppressed because it is too large
@ -0,0 +1,54 @@ |
|||||
|
const fs = require("fs") |
||||
|
const { join } = require("path") |
||||
|
const { spawnSync } =require("child_process") |
||||
|
|
||||
|
const DONT_RUN_PKG = ["bbui"] |
||||
|
const PACKAGES_PATH = join(__dirname, "..", "packages") |
||||
|
|
||||
|
function getPackages() { |
||||
|
return fs.readdirSync(PACKAGES_PATH) |
||||
|
} |
||||
|
|
||||
|
function deleteFile(path) { |
||||
|
try { |
||||
|
fs.unlinkSync(path) |
||||
|
} catch (err) { |
||||
|
// don't error, it just doesn't exist
|
||||
|
} |
||||
|
} |
||||
|
|
||||
|
function removeModules(path) { |
||||
|
if (fs.existsSync(path)) { |
||||
|
fs.rmdirSync(path, { recursive: true }) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
function executeInPackage(packageName) { |
||||
|
if (DONT_RUN_PKG.includes(packageName)) { |
||||
|
return |
||||
|
} |
||||
|
const dir = join(PACKAGES_PATH, packageName) |
||||
|
if (!fs.existsSync(join(dir, "package.json"))) { |
||||
|
console.error(`SKIPPING ${packageName} directory, no package.json`) |
||||
|
return |
||||
|
} |
||||
|
const packageLockLoc = join(dir, "package-lock.json") |
||||
|
const modulesLoc = join(dir, "node_modules") |
||||
|
deleteFile(join(dir, "yarn.lock")) |
||||
|
deleteFile(packageLockLoc) |
||||
|
removeModules(modulesLoc) |
||||
|
const opts = { cwd: dir, stdio: "inherit", shell: true } |
||||
|
spawnSync("npm", ["i", "--package-lock-only"], opts) |
||||
|
spawnSync("npm", ["audit", "fix"], opts) |
||||
|
spawnSync("yarn", ["import"], opts) |
||||
|
deleteFile(packageLockLoc) |
||||
|
removeModules(modulesLoc) |
||||
|
} |
||||
|
|
||||
|
const packages = getPackages() |
||||
|
for (let pkg of packages) { |
||||
|
executeInPackage(pkg) |
||||
|
} |
||||
|
|
||||
|
spawnSync("yarn", ["bootstrap"], { cwd: join(__dirname, ".."), stdio: "inherit", shell: true }) |
||||
|
|
||||
File diff suppressed because it is too large
Loading…
Reference in new issue