mirror of https://github.com/Budibase/budibase.git
Mitch-Budibase
5 years ago
102 changed files with 10263 additions and 8177 deletions
@ -1,3 +1,4 @@ |
|||||
module.exports = { |
module.exports = { |
||||
user: require("./src/cache/user"), |
user: require("./src/cache/user"), |
||||
|
app: require("./src/cache/appMetadata"), |
||||
} |
} |
||||
|
|||||
@ -0,0 +1,85 @@ |
|||||
|
const redis = require("../redis/authRedis") |
||||
|
const { getCouch } = require("../db") |
||||
|
const { DocumentTypes } = require("../db/constants") |
||||
|
|
||||
|
const AppState = { |
||||
|
INVALID: "invalid", |
||||
|
} |
||||
|
const EXPIRY_SECONDS = 3600 |
||||
|
|
||||
|
/** |
||||
|
* The default populate app metadata function |
||||
|
*/ |
||||
|
const populateFromDB = async (appId, CouchDB = null) => { |
||||
|
if (!CouchDB) { |
||||
|
CouchDB = getCouch() |
||||
|
} |
||||
|
const db = new CouchDB(appId, { skip_setup: true }) |
||||
|
return db.get(DocumentTypes.APP_METADATA) |
||||
|
} |
||||
|
|
||||
|
const isInvalid = metadata => { |
||||
|
return !metadata || metadata.state === AppState.INVALID |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* Get the requested app metadata by id. |
||||
|
* Use redis cache to first read the app metadata. |
||||
|
* If not present fallback to loading the app metadata directly and re-caching. |
||||
|
* @param {string} appId the id of the app to get metadata from. |
||||
|
* @param {object} CouchDB the database being passed |
||||
|
* @returns {object} the app metadata. |
||||
|
*/ |
||||
|
exports.getAppMetadata = async (appId, CouchDB = null) => { |
||||
|
const client = await redis.getAppClient() |
||||
|
// try cache
|
||||
|
let metadata = await client.get(appId) |
||||
|
if (!metadata) { |
||||
|
let expiry = EXPIRY_SECONDS |
||||
|
try { |
||||
|
metadata = await populateFromDB(appId, CouchDB) |
||||
|
} catch (err) { |
||||
|
// app DB left around, but no metadata, it is invalid
|
||||
|
if (err && err.status === 404) { |
||||
|
metadata = { state: AppState.INVALID } |
||||
|
// don't expire the reference to an invalid app, it'll only be
|
||||
|
// updated if a metadata doc actually gets stored (app is remade/reverted)
|
||||
|
expiry = undefined |
||||
|
} else { |
||||
|
throw err |
||||
|
} |
||||
|
} |
||||
|
// needed for cypress/some scenarios where the caching happens
|
||||
|
// so quickly the requests can get slightly out of sync
|
||||
|
// might store its invalid just before it stores its valid
|
||||
|
if (isInvalid(metadata)) { |
||||
|
const temp = await client.get(appId) |
||||
|
if (temp) { |
||||
|
metadata = temp |
||||
|
} |
||||
|
} |
||||
|
await client.store(appId, metadata, expiry) |
||||
|
} |
||||
|
// we've stored in the cache an object to tell us that it is currently invalid
|
||||
|
if (isInvalid(metadata)) { |
||||
|
throw { status: 404, message: "No app metadata found" } |
||||
|
} |
||||
|
return metadata |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* Invalidate/reset the cached metadata when a change occurs in the db. |
||||
|
* @param appId {string} the cache key to bust/update. |
||||
|
* @param newMetadata {object|undefined} optional - can simply provide the new metadata to update with. |
||||
|
* @return {Promise<void>} will respond with success when cache is updated. |
||||
|
*/ |
||||
|
exports.invalidateAppMetadata = async (appId, newMetadata = null) => { |
||||
|
if (!appId) { |
||||
|
throw "Cannot invalidate if no app ID provided." |
||||
|
} |
||||
|
const client = await redis.getAppClient() |
||||
|
await client.delete(appId) |
||||
|
if (newMetadata) { |
||||
|
await client.store(appId, newMetadata, EXPIRY_SECONDS) |
||||
|
} |
||||
|
} |
||||
File diff suppressed because it is too large
@ -1,16 +1,67 @@ |
|||||
<script> |
<script> |
||||
import "@spectrum-css/fieldlabel/dist/index-vars.css" |
import "@spectrum-css/fieldlabel/dist/index-vars.css" |
||||
|
import Tooltip from "../Tooltip/Tooltip.svelte" |
||||
|
import Icon from "../Icon/Icon.svelte" |
||||
|
|
||||
export let size = "M" |
export let size = "M" |
||||
|
export let tooltip = "" |
||||
|
export let showTooltip = false |
||||
</script> |
</script> |
||||
|
|
||||
<label for="" class={`spectrum-FieldLabel spectrum-FieldLabel--size${size}`}> |
{#if tooltip} |
||||
<slot /> |
<div class="container"> |
||||
</label> |
<label |
||||
|
for="" |
||||
|
class={`spectrum-FieldLabel spectrum-FieldLabel--size${size}`} |
||||
|
> |
||||
|
<slot /> |
||||
|
</label> |
||||
|
<div class="icon-container"> |
||||
|
<div |
||||
|
class="icon" |
||||
|
on:mouseover={() => (showTooltip = true)} |
||||
|
on:mouseleave={() => (showTooltip = false)} |
||||
|
> |
||||
|
<Icon name="InfoOutline" size="S" disabled={true} /> |
||||
|
</div> |
||||
|
{#if showTooltip} |
||||
|
<div class="tooltip"> |
||||
|
<Tooltip textWrapping={true} direction={"bottom"} text={tooltip} /> |
||||
|
</div> |
||||
|
{/if} |
||||
|
</div> |
||||
|
</div> |
||||
|
{:else} |
||||
|
<label for="" class={`spectrum-FieldLabel spectrum-FieldLabel--size${size}`}> |
||||
|
<slot /> |
||||
|
</label> |
||||
|
{/if} |
||||
|
|
||||
<style> |
<style> |
||||
label { |
label { |
||||
padding: 0; |
padding: 0; |
||||
white-space: nowrap; |
white-space: nowrap; |
||||
} |
} |
||||
|
.container { |
||||
|
display: flex; |
||||
|
} |
||||
|
.icon-container { |
||||
|
position: relative; |
||||
|
display: flex; |
||||
|
justify-content: center; |
||||
|
margin-top: 1px; |
||||
|
margin-left: 5px; |
||||
|
margin-right: 5px; |
||||
|
} |
||||
|
.tooltip { |
||||
|
position: absolute; |
||||
|
display: flex; |
||||
|
justify-content: center; |
||||
|
top: 15px; |
||||
|
z-index: 1; |
||||
|
width: 160px; |
||||
|
} |
||||
|
.icon { |
||||
|
transform: scale(0.75); |
||||
|
} |
||||
</style> |
</style> |
||||
|
|||||
File diff suppressed because it is too large
File diff suppressed because it is too large
@ -0,0 +1,61 @@ |
|||||
|
<script> |
||||
|
import BlockComponent from "components/BlockComponent.svelte" |
||||
|
import Block from "components/Block.svelte" |
||||
|
import Placeholder from "components/app/Placeholder.svelte" |
||||
|
import { getContext } from "svelte" |
||||
|
import { makePropSafe as safe } from "@budibase/string-templates" |
||||
|
|
||||
|
export let dataSource |
||||
|
export let filter |
||||
|
export let sortColumn |
||||
|
export let sortOrder |
||||
|
export let limit |
||||
|
export let paginate |
||||
|
export let noRowsMessage |
||||
|
export let direction |
||||
|
export let hAlign |
||||
|
export let vAlign |
||||
|
export let gap |
||||
|
|
||||
|
let providerId |
||||
|
|
||||
|
const component = getContext("component") |
||||
|
const { styleable } = getContext("sdk") |
||||
|
</script> |
||||
|
|
||||
|
<Block> |
||||
|
<div use:styleable={$component.styles}> |
||||
|
<BlockComponent |
||||
|
type="dataprovider" |
||||
|
context="provider" |
||||
|
bind:id={providerId} |
||||
|
props={{ |
||||
|
dataSource, |
||||
|
filter, |
||||
|
sortColumn, |
||||
|
sortOrder, |
||||
|
limit, |
||||
|
paginate, |
||||
|
}} |
||||
|
> |
||||
|
{#if $component.empty} |
||||
|
<Placeholder text={$component.name} /> |
||||
|
{:else} |
||||
|
<BlockComponent |
||||
|
type="repeater" |
||||
|
context="repeater" |
||||
|
props={{ |
||||
|
dataProvider: `{{ literal ${safe(providerId)} }}`, |
||||
|
noRowsMessage, |
||||
|
direction, |
||||
|
hAlign, |
||||
|
vAlign, |
||||
|
gap, |
||||
|
}} |
||||
|
> |
||||
|
<slot /> |
||||
|
</BlockComponent> |
||||
|
{/if} |
||||
|
</BlockComponent> |
||||
|
</div> |
||||
|
</Block> |
||||
@ -1,2 +1,3 @@ |
|||||
export { default as tableblock } from "./TableBlock.svelte" |
export { default as tableblock } from "./TableBlock.svelte" |
||||
export { default as cardsblock } from "./CardsBlock.svelte" |
export { default as cardsblock } from "./CardsBlock.svelte" |
||||
|
export { default as repeaterblock } from "./RepeaterBlock.svelte" |
||||
|
|||||
File diff suppressed because it is too large
@ -0,0 +1,33 @@ |
|||||
|
/****************************************************** |
||||
|
* This script just makes it easy to re-create * |
||||
|
* a cypress like environment for testing the backend * |
||||
|
******************************************************/ |
||||
|
const path = require("path") |
||||
|
const tmpdir = path.join(require("os").tmpdir(), ".budibase") |
||||
|
|
||||
|
const MAIN_PORT = "10001" |
||||
|
const WORKER_PORT = "10002" |
||||
|
|
||||
|
// @ts-ignore
|
||||
|
process.env.PORT = MAIN_PORT |
||||
|
process.env.BUDIBASE_API_KEY = "6BE826CB-6B30-4AEC-8777-2E90464633DE" |
||||
|
process.env.NODE_ENV = "cypress" |
||||
|
process.env.ENABLE_ANALYTICS = "false" |
||||
|
process.env.JWT_SECRET = "budibase" |
||||
|
process.env.COUCH_URL = `leveldb://${tmpdir}/.data/` |
||||
|
process.env.SELF_HOSTED = "1" |
||||
|
process.env.WORKER_URL = `http://localhost:${WORKER_PORT}/` |
||||
|
process.env.MINIO_URL = `http://localhost:${MAIN_PORT}/` |
||||
|
process.env.MINIO_ACCESS_KEY = "budibase" |
||||
|
process.env.MINIO_SECRET_KEY = "budibase" |
||||
|
process.env.COUCH_DB_USER = "budibase" |
||||
|
process.env.COUCH_DB_PASSWORD = "budibase" |
||||
|
process.env.INTERNAL_API_KEY = "budibase" |
||||
|
process.env.ALLOW_DEV_AUTOMATIONS = "1" |
||||
|
|
||||
|
// don't make this a variable or top level require
|
||||
|
// it will cause environment module to be loaded prematurely
|
||||
|
const server = require("../src/app") |
||||
|
process.env.PORT = WORKER_PORT |
||||
|
const worker = require("../../worker/src/index") |
||||
|
process.env.PORT = MAIN_PORT |
||||
@ -0,0 +1,6 @@ |
|||||
|
export interface IntegrationBase { |
||||
|
create?(query: any): Promise<[any]> |
||||
|
read?(query: any): Promise<[any]> |
||||
|
update?(query: any): Promise<[any]> |
||||
|
delete?(query: any): Promise<[any]> |
||||
|
} |
||||
@ -0,0 +1,60 @@ |
|||||
|
const workerFarm = require("worker-farm") |
||||
|
const env = require("../environment") |
||||
|
|
||||
|
const ThreadType = { |
||||
|
QUERY: "query", |
||||
|
AUTOMATION: "automation", |
||||
|
} |
||||
|
|
||||
|
function typeToFile(type) { |
||||
|
let filename = null |
||||
|
switch (type) { |
||||
|
case ThreadType.QUERY: |
||||
|
filename = "./query" |
||||
|
break |
||||
|
case ThreadType.AUTOMATION: |
||||
|
filename = "./automation" |
||||
|
break |
||||
|
default: |
||||
|
throw "Unknown thread type" |
||||
|
} |
||||
|
return require.resolve(filename) |
||||
|
} |
||||
|
|
||||
|
class Thread { |
||||
|
constructor(type, opts = { timeoutMs: null, count: 1 }) { |
||||
|
this.type = type |
||||
|
if (!env.isTest()) { |
||||
|
const workerOpts = { |
||||
|
autoStart: true, |
||||
|
maxConcurrentWorkers: opts.count ? opts.count : 1, |
||||
|
} |
||||
|
if (opts.timeoutMs) { |
||||
|
workerOpts.maxCallTime = opts.timeoutMs |
||||
|
} |
||||
|
this.workers = workerFarm(workerOpts, typeToFile(type)) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
run(data) { |
||||
|
return new Promise((resolve, reject) => { |
||||
|
let fncToCall |
||||
|
// if in test then don't use threading
|
||||
|
if (env.isTest()) { |
||||
|
fncToCall = require(typeToFile(this.type)) |
||||
|
} else { |
||||
|
fncToCall = this.workers |
||||
|
} |
||||
|
fncToCall(data, (err, response) => { |
||||
|
if (err) { |
||||
|
reject(err) |
||||
|
} else { |
||||
|
resolve(response) |
||||
|
} |
||||
|
}) |
||||
|
}) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
module.exports.Thread = Thread |
||||
|
module.exports.ThreadType = ThreadType |
||||
@ -0,0 +1,63 @@ |
|||||
|
const ScriptRunner = require("../utilities/scriptRunner") |
||||
|
const { integrations } = require("../integrations") |
||||
|
|
||||
|
function formatResponse(resp) { |
||||
|
if (typeof resp === "string") { |
||||
|
try { |
||||
|
resp = JSON.parse(resp) |
||||
|
} catch (err) { |
||||
|
resp = { response: resp } |
||||
|
} |
||||
|
} |
||||
|
return resp |
||||
|
} |
||||
|
|
||||
|
async function runAndTransform(datasource, queryVerb, query, transformer) { |
||||
|
const Integration = integrations[datasource.source] |
||||
|
if (!Integration) { |
||||
|
throw "Integration type does not exist." |
||||
|
} |
||||
|
const integration = new Integration(datasource.config) |
||||
|
|
||||
|
let rows = formatResponse(await integration[queryVerb](query)) |
||||
|
|
||||
|
// transform as required
|
||||
|
if (transformer) { |
||||
|
const runner = new ScriptRunner(transformer, { data: rows }) |
||||
|
rows = runner.execute() |
||||
|
} |
||||
|
|
||||
|
// needs to an array for next step
|
||||
|
if (!Array.isArray(rows)) { |
||||
|
rows = [rows] |
||||
|
} |
||||
|
|
||||
|
// map into JSON if just raw primitive here
|
||||
|
if (rows.find(row => typeof row !== "object")) { |
||||
|
rows = rows.map(value => ({ value })) |
||||
|
} |
||||
|
|
||||
|
// get all the potential fields in the schema
|
||||
|
let keys = rows.flatMap(Object.keys) |
||||
|
|
||||
|
if (integration.end) { |
||||
|
integration.end() |
||||
|
} |
||||
|
|
||||
|
return { rows, keys } |
||||
|
} |
||||
|
|
||||
|
module.exports = (input, callback) => { |
||||
|
runAndTransform( |
||||
|
input.datasource, |
||||
|
input.queryVerb, |
||||
|
input.query, |
||||
|
input.transformer |
||||
|
) |
||||
|
.then(response => { |
||||
|
callback(null, response) |
||||
|
}) |
||||
|
.catch(err => { |
||||
|
callback(err) |
||||
|
}) |
||||
|
} |
||||
File diff suppressed because it is too large
File diff suppressed because it is too large
File diff suppressed because it is too large
Some files were not shown because too many files changed in this diff
Loading…
Reference in new issue