forked from tsai/budibase
Peter Clement
5 years ago
70 changed files with 2359 additions and 704 deletions
@ -0,0 +1,164 @@ |
|||||
|
const env = require("../../environment") |
||||
|
const jwt = require("jsonwebtoken") |
||||
|
const database = require("../../db") |
||||
|
const fetch = require("node-fetch") |
||||
|
const OIDCStrategy = require("@techpass/passport-openidconnect").Strategy |
||||
|
const { |
||||
|
StaticDatabases, |
||||
|
generateGlobalUserID, |
||||
|
ViewNames, |
||||
|
} = require("../../db/utils") |
||||
|
|
||||
|
/** |
||||
|
* Attempt to parse the users email address. |
||||
|
* |
||||
|
* It is not guaranteed that the email will be returned by the user info endpoint (e.g. github connected account used in azure ad). |
||||
|
* Fallback to the id token where possible. |
||||
|
* |
||||
|
* @param {*} profile The structured profile created by passport using the user info endpoint |
||||
|
* @param {*} jwtClaims The raw claims returned in the id token |
||||
|
*/ |
||||
|
function getEmail(profile, jwtClaims) { |
||||
|
if (profile._json.email) { |
||||
|
return profile._json.email |
||||
|
} |
||||
|
|
||||
|
if (jwtClaims.email) { |
||||
|
return jwtClaims.email |
||||
|
} |
||||
|
|
||||
|
return null; |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* |
||||
|
* @param {*} issuer The identity provider base URL |
||||
|
* @param {*} sub The user ID |
||||
|
* @param {*} profile The user profile information. Created by passport from the /userinfo response |
||||
|
* @param {*} jwtClaims The parsed id_token claims |
||||
|
* @param {*} accessToken The access_token for contacting the identity provider - may or may not be a JWT |
||||
|
* @param {*} refreshToken The refresh_token for obtaining a new access_token - usually not a JWT |
||||
|
* @param {*} idToken The id_token - always a JWT |
||||
|
* @param {*} params The response body from requesting an access_token |
||||
|
* @param {*} done The passport callback: err, user, info |
||||
|
* @returns |
||||
|
*/ |
||||
|
async function authenticate( |
||||
|
issuer, |
||||
|
sub, |
||||
|
profile, |
||||
|
jwtClaims, |
||||
|
accessToken, |
||||
|
refreshToken, |
||||
|
idToken, |
||||
|
params, |
||||
|
done |
||||
|
) { |
||||
|
// Check the user exists in the instance DB by email
|
||||
|
const db = database.getDB(StaticDatabases.GLOBAL.name) |
||||
|
|
||||
|
let dbUser |
||||
|
|
||||
|
const userId = generateGlobalUserID(profile.id) |
||||
|
|
||||
|
try { |
||||
|
// use the OIDC profile id
|
||||
|
dbUser = await db.get(userId) |
||||
|
} catch (err) { |
||||
|
const user = { |
||||
|
_id: userId, |
||||
|
provider: profile.provider, |
||||
|
roles: {}, |
||||
|
...profile._json, |
||||
|
} |
||||
|
|
||||
|
// check if an account with the OIDC email address exists locally
|
||||
|
const email = getEmail(profile, jwtClaims) |
||||
|
if (!email) { |
||||
|
return done(null, false, { message: "No email address found" }) |
||||
|
} |
||||
|
|
||||
|
const users = await db.query(`database/${ViewNames.USER_BY_EMAIL}`, { |
||||
|
key: email, |
||||
|
include_docs: true, |
||||
|
}) |
||||
|
|
||||
|
// OIDC user already exists by email
|
||||
|
if (users.rows.length > 0) { |
||||
|
const existing = users.rows[0].doc |
||||
|
|
||||
|
// remove the local account to avoid conflicts
|
||||
|
await db.remove(existing._id, existing._rev) |
||||
|
|
||||
|
// merge with existing account
|
||||
|
user.roles = existing.roles |
||||
|
user.builder = existing.builder |
||||
|
user.admin = existing.admin |
||||
|
|
||||
|
const response = await db.post(user) |
||||
|
dbUser = user |
||||
|
dbUser._rev = response.rev |
||||
|
} else { |
||||
|
return done(null, false, { message: "Email does not yet exist. You must set up your local budibase account first." }) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// authenticate
|
||||
|
const payload = { |
||||
|
userId: dbUser._id, |
||||
|
builder: dbUser.builder, |
||||
|
email: dbUser.email, |
||||
|
} |
||||
|
|
||||
|
dbUser.token = jwt.sign(payload, env.JWT_SECRET, { |
||||
|
expiresIn: "1 day", |
||||
|
}) |
||||
|
|
||||
|
return done(null, dbUser) |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* Create an instance of the oidc passport strategy. This wrapper fetches the configuration |
||||
|
* from couchDB rather than environment variables, using this factory is necessary for dynamically configuring passport. |
||||
|
* @returns Dynamically configured Passport OIDC Strategy |
||||
|
*/ |
||||
|
exports.strategyFactory = async function (callbackUrl) { |
||||
|
try { |
||||
|
const configurationUrl = |
||||
|
"https://login.microsoftonline.com/2668c0dd-7ed2-4db3-b387-05b6f9204a70/v2.0/.well-known/openid-configuration" |
||||
|
const clientSecret = "g-ty~2iW.bo.88xj_QI6~hdc-H8mP2Xbnd" |
||||
|
const clientId = "bed2017b-2f53-42a9-8ef9-e58918935e07" |
||||
|
|
||||
|
if (!clientId || !clientSecret || !callbackUrl || !configurationUrl) { |
||||
|
throw new Error( |
||||
|
"Configuration invalid. Must contain clientID, clientSecret, callbackUrl and configurationUrl" |
||||
|
) |
||||
|
} |
||||
|
|
||||
|
const response = await fetch(configurationUrl) |
||||
|
|
||||
|
if (!response.ok) { |
||||
|
throw new Error(`Unexpected response when fetching openid-configuration: ${response.statusText}`) |
||||
|
} |
||||
|
|
||||
|
const body = await response.json() |
||||
|
|
||||
|
return new OIDCStrategy( |
||||
|
{ |
||||
|
issuer: body.issuer, |
||||
|
authorizationURL: body.authorization_endpoint, |
||||
|
tokenURL: body.token_endpoint, |
||||
|
userInfoURL: body.userinfo_endpoint, |
||||
|
clientID: clientId, |
||||
|
clientSecret: clientSecret, |
||||
|
callbackURL: callbackUrl, |
||||
|
scope: "profile email", |
||||
|
}, |
||||
|
authenticate |
||||
|
) |
||||
|
|
||||
|
} catch (err) { |
||||
|
console.error(err) |
||||
|
throw new Error("Error constructing OIDC authentication strategy", err) |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,194 @@ |
|||||
|
<script> |
||||
|
import { RelationshipTypes } from "constants/backend" |
||||
|
import { Button, Input, ModalContent, Select, Detail } from "@budibase/bbui" |
||||
|
import { tables } from "stores/backend" |
||||
|
import { uuid } from "builderStore/uuid" |
||||
|
|
||||
|
export let save |
||||
|
export let datasource |
||||
|
export let plusTables = [] |
||||
|
export let fromRelationship = {} |
||||
|
export let toRelationship = {} |
||||
|
export let close |
||||
|
|
||||
|
let originalFromName = fromRelationship.name, |
||||
|
originalToName = toRelationship.name |
||||
|
|
||||
|
function isValid(relationship) { |
||||
|
if ( |
||||
|
relationship.relationshipType === RelationshipTypes.MANY_TO_MANY && |
||||
|
!relationship.through |
||||
|
) { |
||||
|
return false |
||||
|
} |
||||
|
return ( |
||||
|
relationship.name && relationship.tableId && relationship.relationshipType |
||||
|
) |
||||
|
} |
||||
|
|
||||
|
$: tableOptions = plusTables.map(table => ({ |
||||
|
label: table.name, |
||||
|
value: table._id, |
||||
|
})) |
||||
|
$: fromTable = plusTables.find(table => table._id === toRelationship?.tableId) |
||||
|
$: toTable = plusTables.find(table => table._id === fromRelationship?.tableId) |
||||
|
$: through = plusTables.find(table => table._id === fromRelationship?.through) |
||||
|
$: valid = toTable && fromTable && isValid(fromRelationship) |
||||
|
$: linkTable = through || toTable |
||||
|
$: relationshipTypes = [ |
||||
|
{ |
||||
|
label: "Many", |
||||
|
value: RelationshipTypes.MANY_TO_MANY, |
||||
|
}, |
||||
|
{ |
||||
|
label: "One", |
||||
|
value: RelationshipTypes.MANY_TO_ONE, |
||||
|
}, |
||||
|
] |
||||
|
$: updateRelationshipType(fromRelationship?.relationshipType) |
||||
|
|
||||
|
function updateRelationshipType(fromType) { |
||||
|
if (fromType === RelationshipTypes.MANY_TO_MANY) { |
||||
|
toRelationship.relationshipType = RelationshipTypes.MANY_TO_MANY |
||||
|
} else { |
||||
|
toRelationship.relationshipType = RelationshipTypes.MANY_TO_ONE |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
function buildRelationships() { |
||||
|
// if any to many only need to check from |
||||
|
const manyToMany = |
||||
|
fromRelationship.relationshipType === RelationshipTypes.MANY_TO_MANY |
||||
|
// main is simply used to know this is the side the user configured it from |
||||
|
const id = uuid() |
||||
|
if (!manyToMany) { |
||||
|
delete fromRelationship.through |
||||
|
delete toRelationship.through |
||||
|
} |
||||
|
let relateFrom = { |
||||
|
...fromRelationship, |
||||
|
type: "link", |
||||
|
main: true, |
||||
|
_id: id, |
||||
|
} |
||||
|
let relateTo = { |
||||
|
...toRelationship, |
||||
|
type: "link", |
||||
|
_id: id, |
||||
|
} |
||||
|
|
||||
|
// [0] is because we don't support composite keys for relationships right now |
||||
|
if (manyToMany) { |
||||
|
relateFrom = { |
||||
|
...relateFrom, |
||||
|
through: through._id, |
||||
|
fieldName: toTable.primary[0], |
||||
|
} |
||||
|
relateTo = { |
||||
|
...relateTo, |
||||
|
through: through._id, |
||||
|
fieldName: fromTable.primary[0], |
||||
|
} |
||||
|
} else { |
||||
|
relateFrom = { |
||||
|
...relateFrom, |
||||
|
foreignKey: relateFrom.fieldName, |
||||
|
fieldName: fromTable.primary[0], |
||||
|
} |
||||
|
relateTo = { |
||||
|
...relateTo, |
||||
|
relationshipType: RelationshipTypes.ONE_TO_MANY, |
||||
|
foreignKey: relateFrom.fieldName, |
||||
|
fieldName: fromTable.primary[0], |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
fromRelationship = relateFrom |
||||
|
toRelationship = relateTo |
||||
|
} |
||||
|
|
||||
|
// save the relationship on to the datasource |
||||
|
async function saveRelationship() { |
||||
|
buildRelationships() |
||||
|
// source of relationship |
||||
|
datasource.entities[fromTable.name].schema[fromRelationship.name] = |
||||
|
fromRelationship |
||||
|
// save other side of relationship in the other schema |
||||
|
datasource.entities[toTable.name].schema[toRelationship.name] = |
||||
|
toRelationship |
||||
|
|
||||
|
// If relationship has been renamed |
||||
|
if (originalFromName !== fromRelationship.name) { |
||||
|
delete datasource.entities[fromTable.name].schema[originalFromName] |
||||
|
} |
||||
|
if (originalToName !== toRelationship.name) { |
||||
|
delete datasource.entities[toTable.name].schema[originalToName] |
||||
|
} |
||||
|
|
||||
|
await save() |
||||
|
await tables.fetch() |
||||
|
} |
||||
|
|
||||
|
async function deleteRelationship() { |
||||
|
delete datasource.entities[fromTable.name].schema[fromRelationship.name] |
||||
|
delete datasource.entities[toTable.name].schema[toRelationship.name] |
||||
|
await save() |
||||
|
await tables.fetch() |
||||
|
close() |
||||
|
} |
||||
|
</script> |
||||
|
|
||||
|
<ModalContent |
||||
|
title="Create Relationship" |
||||
|
confirmText="Save" |
||||
|
onConfirm={saveRelationship} |
||||
|
disabled={!valid} |
||||
|
> |
||||
|
<Select |
||||
|
label="Relationship type" |
||||
|
options={relationshipTypes} |
||||
|
bind:value={fromRelationship.relationshipType} |
||||
|
/> |
||||
|
<div class="headings"> |
||||
|
<Detail>Tables</Detail> |
||||
|
</div> |
||||
|
<Select |
||||
|
label="Select from table" |
||||
|
options={tableOptions} |
||||
|
bind:value={toRelationship.tableId} |
||||
|
/> |
||||
|
<Select |
||||
|
label={"Select to table"} |
||||
|
options={tableOptions} |
||||
|
bind:value={fromRelationship.tableId} |
||||
|
/> |
||||
|
{#if fromRelationship?.relationshipType === RelationshipTypes.MANY_TO_MANY} |
||||
|
<Select |
||||
|
label={"Through"} |
||||
|
options={tableOptions} |
||||
|
bind:value={fromRelationship.through} |
||||
|
/> |
||||
|
{:else if toTable} |
||||
|
<Select |
||||
|
label={`Foreign Key (${toTable?.name})`} |
||||
|
options={Object.keys(toTable?.schema)} |
||||
|
bind:value={fromRelationship.fieldName} |
||||
|
/> |
||||
|
{/if} |
||||
|
<div class="headings"> |
||||
|
<Detail>Column names</Detail> |
||||
|
</div> |
||||
|
<Input label="From table column" bind:value={fromRelationship.name} /> |
||||
|
<Input label="To table column" bind:value={toRelationship.name} /> |
||||
|
<div slot="footer"> |
||||
|
{#if originalFromName != null} |
||||
|
<Button warning text on:click={deleteRelationship}>Delete</Button> |
||||
|
{/if} |
||||
|
</div> |
||||
|
</ModalContent> |
||||
|
|
||||
|
<style> |
||||
|
.headings { |
||||
|
margin-top: var(--spacing-s); |
||||
|
} |
||||
|
</style> |
||||
@ -0,0 +1,21 @@ |
|||||
|
<script> |
||||
|
import { Menu, Icon, MenuSection, MenuItem } from "@budibase/bbui" |
||||
|
|
||||
|
export let heading |
||||
|
export let tables |
||||
|
export let selected = false |
||||
|
export let select |
||||
|
</script> |
||||
|
|
||||
|
<Menu> |
||||
|
<MenuSection {heading}> |
||||
|
{#each tables as table} |
||||
|
<MenuItem noClose icon="Table" on:click={() => select(table)}> |
||||
|
{table.name} |
||||
|
{#if selected} |
||||
|
<Icon size="S" name="Checkmark" /> |
||||
|
{/if} |
||||
|
</MenuItem> |
||||
|
{/each} |
||||
|
</MenuSection> |
||||
|
</Menu> |
||||
@ -0,0 +1,43 @@ |
|||||
|
<script> |
||||
|
import { ModalContent, Select, Body } from "@budibase/bbui" |
||||
|
import { tables } from "stores/backend" |
||||
|
|
||||
|
export let datasource |
||||
|
export let plusTables |
||||
|
export let save |
||||
|
|
||||
|
async function saveDisplayColumns() { |
||||
|
// be explicit about copying over |
||||
|
for (let table of plusTables) { |
||||
|
datasource.entities[table.name].primaryDisplay = table.primaryDisplay |
||||
|
} |
||||
|
save() |
||||
|
await tables.fetch() |
||||
|
} |
||||
|
|
||||
|
function getColumnOptions(table) { |
||||
|
if (!table || !table.schema) { |
||||
|
return [] |
||||
|
} |
||||
|
return Object.entries(table.schema) |
||||
|
.filter(field => field[1].type !== "link") |
||||
|
.map(([fieldName]) => fieldName) |
||||
|
} |
||||
|
</script> |
||||
|
|
||||
|
<ModalContent |
||||
|
title="Edit display columns" |
||||
|
confirmText="Save" |
||||
|
onConfirm={saveDisplayColumns} |
||||
|
> |
||||
|
<Body |
||||
|
>Select the columns that will be shown when displaying relationships.</Body |
||||
|
> |
||||
|
{#each plusTables as table} |
||||
|
<Select |
||||
|
label={table.name} |
||||
|
options={getColumnOptions(table)} |
||||
|
bind:value={table.primaryDisplay} |
||||
|
/> |
||||
|
{/each} |
||||
|
</ModalContent> |
||||
@ -0,0 +1,32 @@ |
|||||
|
<script> |
||||
|
import { ActionButton } from "@budibase/bbui" |
||||
|
// import { admin } from "stores/portal" |
||||
|
|
||||
|
let show = true |
||||
|
</script> |
||||
|
|
||||
|
{#if show} |
||||
|
<ActionButton on:click={() => window.open("/api/admin/auth/oidc", "_blank")}> |
||||
|
<div class="inner"> |
||||
|
<p>Sign in with OIDC</p> |
||||
|
</div> |
||||
|
</ActionButton> |
||||
|
{/if} |
||||
|
|
||||
|
<style> |
||||
|
.inner { |
||||
|
display: flex; |
||||
|
flex-direction: row; |
||||
|
align-items: center; |
||||
|
justify-content: center; |
||||
|
padding-top: var(--spacing-xs); |
||||
|
padding-bottom: var(--spacing-xs); |
||||
|
} |
||||
|
/* .inner img { |
||||
|
width: 18px; |
||||
|
margin: 3px 10px 3px 3px; |
||||
|
} */ |
||||
|
.inner p { |
||||
|
margin: 0; |
||||
|
} |
||||
|
</style> |
||||
File diff suppressed because it is too large
@ -0,0 +1,3 @@ |
|||||
|
#!/bin/bash |
||||
|
docker-compose down |
||||
|
docker volume prune -f |
||||
@ -0,0 +1,525 @@ |
|||||
|
import { |
||||
|
Operation, |
||||
|
SearchFilters, |
||||
|
SortJson, |
||||
|
PaginationJson, |
||||
|
RelationshipsJson, |
||||
|
} from "../../../definitions/datasource" |
||||
|
import { |
||||
|
Row, |
||||
|
Table, |
||||
|
FieldSchema, |
||||
|
Datasource, |
||||
|
} from "../../../definitions/common" |
||||
|
import { |
||||
|
breakRowIdField, |
||||
|
generateRowIdField, |
||||
|
} from "../../../integrations/utils" |
||||
|
|
||||
|
interface ManyRelationship { |
||||
|
tableId?: string |
||||
|
id?: string |
||||
|
isUpdate?: boolean |
||||
|
[key: string]: any |
||||
|
} |
||||
|
|
||||
|
interface RunConfig { |
||||
|
id: string |
||||
|
row: Row |
||||
|
filters: SearchFilters |
||||
|
sort: SortJson |
||||
|
paginate: PaginationJson |
||||
|
} |
||||
|
|
||||
|
module External { |
||||
|
const { makeExternalQuery } = require("./utils") |
||||
|
const { DataSourceOperation, FieldTypes } = require("../../../constants") |
||||
|
const { breakExternalTableId, isSQL } = require("../../../integrations/utils") |
||||
|
const { processObjectSync } = require("@budibase/string-templates") |
||||
|
const { cloneDeep } = require("lodash/fp") |
||||
|
const { isEqual } = require("lodash") |
||||
|
const CouchDB = require("../../../db") |
||||
|
|
||||
|
function buildFilters( |
||||
|
id: string | undefined, |
||||
|
filters: SearchFilters, |
||||
|
table: Table |
||||
|
) { |
||||
|
const primary = table.primary |
||||
|
// if passed in array need to copy for shifting etc
|
||||
|
let idCopy = cloneDeep(id) |
||||
|
if (filters) { |
||||
|
// need to map over the filters and make sure the _id field isn't present
|
||||
|
for (let filter of Object.values(filters)) { |
||||
|
if (filter._id && primary) { |
||||
|
const parts = breakRowIdField(filter._id) |
||||
|
for (let field of primary) { |
||||
|
filter[field] = parts.shift() |
||||
|
} |
||||
|
} |
||||
|
// make sure this field doesn't exist on any filter
|
||||
|
delete filter._id |
||||
|
} |
||||
|
} |
||||
|
// there is no id, just use the user provided filters
|
||||
|
if (!idCopy || !table) { |
||||
|
return filters |
||||
|
} |
||||
|
// if used as URL parameter it will have been joined
|
||||
|
if (!Array.isArray(idCopy)) { |
||||
|
idCopy = breakRowIdField(idCopy) |
||||
|
} |
||||
|
const equal: any = {} |
||||
|
if (primary && idCopy) { |
||||
|
for (let field of primary) { |
||||
|
// work through the ID and get the parts
|
||||
|
equal[field] = idCopy.shift() |
||||
|
} |
||||
|
} |
||||
|
return { |
||||
|
equal, |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
function generateIdForRow(row: Row, table: Table): string { |
||||
|
const primary = table.primary |
||||
|
if (!row || !primary) { |
||||
|
return "" |
||||
|
} |
||||
|
// build id array
|
||||
|
let idParts = [] |
||||
|
for (let field of primary) { |
||||
|
if (row[field]) { |
||||
|
idParts.push(row[field]) |
||||
|
} |
||||
|
} |
||||
|
if (idParts.length === 0) { |
||||
|
return "" |
||||
|
} |
||||
|
return generateRowIdField(idParts) |
||||
|
} |
||||
|
|
||||
|
function getEndpoint(tableId: string | undefined, operation: string) { |
||||
|
if (!tableId) { |
||||
|
return {} |
||||
|
} |
||||
|
const { datasourceId, tableName } = breakExternalTableId(tableId) |
||||
|
return { |
||||
|
datasourceId, |
||||
|
entityId: tableName, |
||||
|
operation, |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
function basicProcessing(row: Row, table: Table) { |
||||
|
const thisRow: { [key: string]: any } = {} |
||||
|
// filter the row down to what is actually the row (not joined)
|
||||
|
for (let fieldName of Object.keys(table.schema)) { |
||||
|
thisRow[fieldName] = row[fieldName] |
||||
|
} |
||||
|
thisRow._id = generateIdForRow(row, table) |
||||
|
thisRow.tableId = table._id |
||||
|
thisRow._rev = "rev" |
||||
|
return thisRow |
||||
|
} |
||||
|
|
||||
|
function isMany(field: FieldSchema) { |
||||
|
return ( |
||||
|
field.relationshipType && field.relationshipType.split("-")[0] === "many" |
||||
|
) |
||||
|
} |
||||
|
|
||||
|
class ExternalRequest { |
||||
|
private readonly appId: string |
||||
|
private operation: Operation |
||||
|
private tableId: string |
||||
|
private datasource: Datasource |
||||
|
private tables: { [key: string]: Table } = {} |
||||
|
|
||||
|
constructor( |
||||
|
appId: string, |
||||
|
operation: Operation, |
||||
|
tableId: string, |
||||
|
datasource: Datasource |
||||
|
) { |
||||
|
this.appId = appId |
||||
|
this.operation = operation |
||||
|
this.tableId = tableId |
||||
|
this.datasource = datasource |
||||
|
if (datasource && datasource.entities) { |
||||
|
this.tables = datasource.entities |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
inputProcessing(row: Row, table: Table) { |
||||
|
if (!row) { |
||||
|
return { row, manyRelationships: [] } |
||||
|
} |
||||
|
// we don't really support composite keys for relationships, this is why [0] is used
|
||||
|
// @ts-ignore
|
||||
|
const tablePrimary: string = table.primary[0] |
||||
|
let newRow: Row = {}, |
||||
|
manyRelationships: ManyRelationship[] = [] |
||||
|
for (let [key, field] of Object.entries(table.schema)) { |
||||
|
// if set already, or not set just skip it
|
||||
|
if (!row[key] || newRow[key]) { |
||||
|
continue |
||||
|
} |
||||
|
// if its not a link then just copy it over
|
||||
|
if (field.type !== FieldTypes.LINK) { |
||||
|
newRow[key] = row[key] |
||||
|
continue |
||||
|
} |
||||
|
const { tableName: linkTableName } = breakExternalTableId(field.tableId) |
||||
|
// table has to exist for many to many
|
||||
|
if (!this.tables[linkTableName]) { |
||||
|
continue |
||||
|
} |
||||
|
const linkTable = this.tables[linkTableName] |
||||
|
// @ts-ignore
|
||||
|
const linkTablePrimary = linkTable.primary[0] |
||||
|
if (!isMany(field)) { |
||||
|
newRow[field.foreignKey || linkTablePrimary] = breakRowIdField( |
||||
|
row[key][0] |
||||
|
)[0] |
||||
|
} else { |
||||
|
// we're not inserting a doc, will be a bunch of update calls
|
||||
|
const isUpdate = !field.through |
||||
|
const thisKey: string = isUpdate ? "id" : linkTablePrimary |
||||
|
// @ts-ignore
|
||||
|
const otherKey: string = isUpdate ? field.foreignKey : tablePrimary |
||||
|
row[key].map((relationship: any) => { |
||||
|
// we don't really support composite keys for relationships, this is why [0] is used
|
||||
|
manyRelationships.push({ |
||||
|
tableId: field.through || field.tableId, |
||||
|
isUpdate, |
||||
|
[thisKey]: breakRowIdField(relationship)[0], |
||||
|
// leave the ID for enrichment later
|
||||
|
[otherKey]: `{{ literal ${tablePrimary} }}`, |
||||
|
}) |
||||
|
}) |
||||
|
} |
||||
|
} |
||||
|
// we return the relationships that may need to be created in the through table
|
||||
|
// we do this so that if the ID is generated by the DB it can be inserted
|
||||
|
// after the fact
|
||||
|
return { row: newRow, manyRelationships } |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* This iterates through the returned rows and works out what elements of the rows |
||||
|
* actually match up to another row (based on primary keys) - this is pretty specific |
||||
|
* to SQL and the way that SQL relationships are returned based on joins. |
||||
|
*/ |
||||
|
updateRelationshipColumns( |
||||
|
row: Row, |
||||
|
rows: { [key: string]: Row }, |
||||
|
relationships: RelationshipsJson[] |
||||
|
) { |
||||
|
const columns: { [key: string]: any } = {} |
||||
|
for (let relationship of relationships) { |
||||
|
const linkedTable = this.tables[relationship.tableName] |
||||
|
if (!linkedTable) { |
||||
|
continue |
||||
|
} |
||||
|
let linked = basicProcessing(row, linkedTable) |
||||
|
if (!linked._id) { |
||||
|
continue |
||||
|
} |
||||
|
// if not returning full docs then get the minimal links out
|
||||
|
const display = linkedTable.primaryDisplay |
||||
|
linked = { |
||||
|
primaryDisplay: display ? linked[display] : undefined, |
||||
|
_id: linked._id, |
||||
|
} |
||||
|
columns[relationship.column] = linked |
||||
|
} |
||||
|
for (let [column, related] of Object.entries(columns)) { |
||||
|
if (!row._id) { |
||||
|
continue |
||||
|
} |
||||
|
const rowId: string = row._id |
||||
|
if (!Array.isArray(rows[rowId][column])) { |
||||
|
rows[rowId][column] = [] |
||||
|
} |
||||
|
// make sure relationship hasn't been found already
|
||||
|
if ( |
||||
|
!rows[rowId][column].find( |
||||
|
(relation: Row) => relation._id === related._id |
||||
|
) |
||||
|
) { |
||||
|
rows[rowId][column].push(related) |
||||
|
} |
||||
|
} |
||||
|
return rows |
||||
|
} |
||||
|
|
||||
|
outputProcessing( |
||||
|
rows: Row[], |
||||
|
table: Table, |
||||
|
relationships: RelationshipsJson[] |
||||
|
) { |
||||
|
if (rows[0].read === true) { |
||||
|
return [] |
||||
|
} |
||||
|
let finalRows: { [key: string]: Row } = {} |
||||
|
for (let row of rows) { |
||||
|
const rowId = generateIdForRow(row, table) |
||||
|
row._id = rowId |
||||
|
// this is a relationship of some sort
|
||||
|
if (finalRows[rowId]) { |
||||
|
finalRows = this.updateRelationshipColumns( |
||||
|
row, |
||||
|
finalRows, |
||||
|
relationships |
||||
|
) |
||||
|
continue |
||||
|
} |
||||
|
const thisRow = basicProcessing(row, table) |
||||
|
finalRows[thisRow._id] = thisRow |
||||
|
// do this at end once its been added to the final rows
|
||||
|
finalRows = this.updateRelationshipColumns( |
||||
|
row, |
||||
|
finalRows, |
||||
|
relationships |
||||
|
) |
||||
|
} |
||||
|
return Object.values(finalRows) |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* Gets the list of relationship JSON structures based on the columns in the table, |
||||
|
* this will be used by the underlying library to build whatever relationship mechanism |
||||
|
* it has (e.g. SQL joins). |
||||
|
*/ |
||||
|
buildRelationships(table: Table): RelationshipsJson[] { |
||||
|
const relationships = [] |
||||
|
for (let [fieldName, field] of Object.entries(table.schema)) { |
||||
|
if (field.type !== FieldTypes.LINK) { |
||||
|
continue |
||||
|
} |
||||
|
const { tableName: linkTableName } = breakExternalTableId(field.tableId) |
||||
|
// no table to link to, this is not a valid relationships
|
||||
|
if (!this.tables[linkTableName]) { |
||||
|
continue |
||||
|
} |
||||
|
const linkTable = this.tables[linkTableName] |
||||
|
if (!table.primary || !linkTable.primary) { |
||||
|
continue |
||||
|
} |
||||
|
const definition = { |
||||
|
// if no foreign key specified then use the name of the field in other table
|
||||
|
from: field.foreignKey || table.primary[0], |
||||
|
to: field.fieldName, |
||||
|
tableName: linkTableName, |
||||
|
through: undefined, |
||||
|
// need to specify where to put this back into
|
||||
|
column: fieldName, |
||||
|
} |
||||
|
if (field.through) { |
||||
|
const { tableName: throughTableName } = breakExternalTableId( |
||||
|
field.through |
||||
|
) |
||||
|
definition.through = throughTableName |
||||
|
// don't support composite keys for relationships
|
||||
|
definition.from = table.primary[0] |
||||
|
definition.to = linkTable.primary[0] |
||||
|
} |
||||
|
relationships.push(definition) |
||||
|
} |
||||
|
return relationships |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* This is a cached lookup, of relationship records, this is mainly for creating/deleting junction |
||||
|
* information. |
||||
|
*/ |
||||
|
async lookup( |
||||
|
row: Row, |
||||
|
relationship: ManyRelationship, |
||||
|
cache: { [key: string]: Row[] } = {} |
||||
|
) { |
||||
|
const { tableId, isUpdate, id, ...rest } = relationship |
||||
|
const { tableName } = breakExternalTableId(tableId) |
||||
|
const table = this.tables[tableName] |
||||
|
if (isUpdate) { |
||||
|
return { rows: [], table } |
||||
|
} |
||||
|
// if not updating need to make sure we have a list of all possible options
|
||||
|
let fullKey: string = tableId + "/", |
||||
|
rowKey: string = "" |
||||
|
for (let key of Object.keys(rest)) { |
||||
|
if (row[key]) { |
||||
|
fullKey += key |
||||
|
rowKey = key |
||||
|
} |
||||
|
} |
||||
|
if (cache[fullKey] == null) { |
||||
|
cache[fullKey] = await makeExternalQuery(this.appId, { |
||||
|
endpoint: getEndpoint(tableId, DataSourceOperation.READ), |
||||
|
filters: { |
||||
|
equal: { |
||||
|
[rowKey]: row[rowKey], |
||||
|
}, |
||||
|
}, |
||||
|
}) |
||||
|
} |
||||
|
return { rows: cache[fullKey], table } |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* Once a row has been written we may need to update a many field, e.g. updating foreign keys |
||||
|
* in a bunch of rows in another table, or inserting/deleting rows from a junction table (many to many). |
||||
|
* This is quite a complex process and is handled by this function, there are a few things going on here: |
||||
|
* 1. If updating foreign keys its relatively simple, just create a filter for the row that needs updated |
||||
|
* and write the various components. |
||||
|
* 2. If junction table, then we lookup what exists already, write what doesn't exist, work out what |
||||
|
* isn't supposed to exist anymore and delete those. This is better than the usual method of delete them |
||||
|
* all and then re-create, as theres no chance of losing data (e.g. delete succeed, but write fail). |
||||
|
*/ |
||||
|
async handleManyRelationships(row: Row, relationships: ManyRelationship[]) { |
||||
|
const { appId } = this |
||||
|
if (relationships.length === 0) { |
||||
|
return |
||||
|
} |
||||
|
// if we're creating (in a through table) need to wipe the existing ones first
|
||||
|
const promises = [] |
||||
|
const cache: { [key: string]: Row[] } = {} |
||||
|
for (let relationship of relationships) { |
||||
|
const { tableId, isUpdate, id, ...rest } = relationship |
||||
|
const body = processObjectSync(rest, row) |
||||
|
const { table, rows } = await this.lookup(row, relationship, cache) |
||||
|
const found = rows.find(row => isEqual(body, row)) |
||||
|
const operation = isUpdate |
||||
|
? DataSourceOperation.UPDATE |
||||
|
: DataSourceOperation.CREATE |
||||
|
if (!found) { |
||||
|
promises.push( |
||||
|
makeExternalQuery(appId, { |
||||
|
endpoint: getEndpoint(tableId, operation), |
||||
|
// if we're doing many relationships then we're writing, only one response
|
||||
|
body, |
||||
|
filters: buildFilters(id, {}, table), |
||||
|
}) |
||||
|
) |
||||
|
} else { |
||||
|
// remove the relationship from the rows
|
||||
|
rows.splice(rows.indexOf(found), 1) |
||||
|
} |
||||
|
} |
||||
|
// finally if creating, cleanup any rows that aren't supposed to be here
|
||||
|
for (let [key, rows] of Object.entries(cache)) { |
||||
|
// @ts-ignore
|
||||
|
const tableId: string = key.split("/").shift() |
||||
|
const { tableName } = breakExternalTableId(tableId) |
||||
|
const table = this.tables[tableName] |
||||
|
for (let row of rows) { |
||||
|
promises.push( |
||||
|
makeExternalQuery(this.appId, { |
||||
|
endpoint: getEndpoint(tableId, DataSourceOperation.DELETE), |
||||
|
filters: buildFilters(generateIdForRow(row, table), {}, table), |
||||
|
}) |
||||
|
) |
||||
|
} |
||||
|
} |
||||
|
await Promise.all(promises) |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* This function is a bit crazy, but the exact purpose of it is to protect against the scenario in which |
||||
|
* you have column overlap in relationships, e.g. we join a few different tables and they all have the |
||||
|
* concept of an ID, but for some of them it will be null (if they say don't have a relationship). |
||||
|
* Creating the specific list of fields that we desire, and excluding the ones that are no use to us |
||||
|
* is more performant and has the added benefit of protecting against this scenario. |
||||
|
*/ |
||||
|
buildFields(table: Table) { |
||||
|
function extractNonLinkFieldNames(table: Table, existing: string[] = []) { |
||||
|
return Object.entries(table.schema) |
||||
|
.filter( |
||||
|
column => |
||||
|
column[1].type !== FieldTypes.LINK && |
||||
|
!existing.find((field: string) => field.includes(column[0])) |
||||
|
) |
||||
|
.map(column => `${table.name}.${column[0]}`) |
||||
|
} |
||||
|
let fields = extractNonLinkFieldNames(table) |
||||
|
for (let field of Object.values(table.schema)) { |
||||
|
if (field.type !== FieldTypes.LINK) { |
||||
|
continue |
||||
|
} |
||||
|
const { tableName: linkTableName } = breakExternalTableId(field.tableId) |
||||
|
const linkTable = this.tables[linkTableName] |
||||
|
if (linkTable) { |
||||
|
const linkedFields = extractNonLinkFieldNames(linkTable, fields) |
||||
|
fields = fields.concat(linkedFields) |
||||
|
} |
||||
|
} |
||||
|
return fields |
||||
|
} |
||||
|
|
||||
|
async run({ id, row, filters, sort, paginate }: RunConfig) { |
||||
|
const { appId, operation, tableId } = this |
||||
|
let { datasourceId, tableName } = breakExternalTableId(tableId) |
||||
|
if (!this.datasource) { |
||||
|
const db = new CouchDB(appId) |
||||
|
this.datasource = await db.get(datasourceId) |
||||
|
if (!this.datasource || !this.datasource.entities) { |
||||
|
throw "No tables found, fetch tables before query." |
||||
|
} |
||||
|
this.tables = this.datasource.entities |
||||
|
} |
||||
|
const table = this.tables[tableName] |
||||
|
let isSql = isSQL(this.datasource) |
||||
|
if (!table) { |
||||
|
throw `Unable to process query, table "${tableName}" not defined.` |
||||
|
} |
||||
|
// clean up row on ingress using schema
|
||||
|
filters = buildFilters(id, filters, table) |
||||
|
const relationships = this.buildRelationships(table) |
||||
|
const processed = this.inputProcessing(row, table) |
||||
|
row = processed.row |
||||
|
if ( |
||||
|
operation === DataSourceOperation.DELETE && |
||||
|
(filters == null || Object.keys(filters).length === 0) |
||||
|
) { |
||||
|
throw "Deletion must be filtered" |
||||
|
} |
||||
|
let json = { |
||||
|
endpoint: { |
||||
|
datasourceId, |
||||
|
entityId: tableName, |
||||
|
operation, |
||||
|
}, |
||||
|
resource: { |
||||
|
// have to specify the fields to avoid column overlap (for SQL)
|
||||
|
fields: isSql ? this.buildFields(table) : [], |
||||
|
}, |
||||
|
filters, |
||||
|
sort, |
||||
|
paginate, |
||||
|
relationships, |
||||
|
body: row, |
||||
|
// pass an id filter into extra, purely for mysql/returning
|
||||
|
extra: { |
||||
|
idFilter: buildFilters(id || generateIdForRow(row, table), {}, table), |
||||
|
}, |
||||
|
} |
||||
|
// can't really use response right now
|
||||
|
const response = await makeExternalQuery(appId, json) |
||||
|
// handle many to many relationships now if we know the ID (could be auto increment)
|
||||
|
if (processed.manyRelationships) { |
||||
|
await this.handleManyRelationships( |
||||
|
response[0], |
||||
|
processed.manyRelationships |
||||
|
) |
||||
|
} |
||||
|
const output = this.outputProcessing(response, table, relationships) |
||||
|
// if reading it'll just be an array of rows, return whole thing
|
||||
|
return operation === DataSourceOperation.READ && Array.isArray(response) |
||||
|
? output |
||||
|
: { row: output[0], table } |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
module.exports = ExternalRequest |
||||
|
} |
||||
@ -0,0 +1,100 @@ |
|||||
|
import { SourceNames } from "./datasource" |
||||
|
|
||||
|
interface Base { |
||||
|
_id?: string |
||||
|
_rev?: string |
||||
|
} |
||||
|
|
||||
|
export interface FieldSchema { |
||||
|
// TODO: replace with field types enum when done
|
||||
|
type: string |
||||
|
fieldName?: string |
||||
|
name: string |
||||
|
tableId?: string |
||||
|
relationshipType?: string |
||||
|
through?: string |
||||
|
foreignKey?: string |
||||
|
constraints?: { |
||||
|
type?: string |
||||
|
email?: boolean |
||||
|
inclusion?: string[] |
||||
|
length?: { |
||||
|
minimum?: string | number |
||||
|
maximum?: string | number |
||||
|
} |
||||
|
presence?: boolean |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
export interface TableSchema { |
||||
|
[key: string]: FieldSchema |
||||
|
} |
||||
|
|
||||
|
export interface Table extends Base { |
||||
|
type?: string |
||||
|
views?: {} |
||||
|
name?: string |
||||
|
primary?: string[] |
||||
|
schema: TableSchema |
||||
|
primaryDisplay?: string |
||||
|
sourceId?: string |
||||
|
} |
||||
|
|
||||
|
export interface Row extends Base { |
||||
|
type?: string |
||||
|
tableId?: string |
||||
|
[key: string]: any |
||||
|
} |
||||
|
|
||||
|
interface JsonSchemaField { |
||||
|
properties: { |
||||
|
[key: string]: { |
||||
|
type: string |
||||
|
title: string |
||||
|
customType?: string |
||||
|
} |
||||
|
} |
||||
|
required?: string[] |
||||
|
} |
||||
|
|
||||
|
export interface AutomationStep { |
||||
|
description: string |
||||
|
event?: string |
||||
|
icon: string |
||||
|
id: string |
||||
|
inputs: { |
||||
|
[key: string]: any |
||||
|
} |
||||
|
name: string |
||||
|
schema: { |
||||
|
inputs: JsonSchemaField |
||||
|
outputs: JsonSchemaField |
||||
|
} |
||||
|
stepId: string |
||||
|
tagline: string |
||||
|
type: string |
||||
|
} |
||||
|
|
||||
|
export interface Automation extends Base { |
||||
|
name: string |
||||
|
type: string |
||||
|
appId?: string |
||||
|
definition: { |
||||
|
steps: AutomationStep[] |
||||
|
trigger?: AutomationStep |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
export interface Datasource extends Base { |
||||
|
type: string |
||||
|
name: string |
||||
|
source: SourceNames |
||||
|
// the config is defined by the schema
|
||||
|
config: { |
||||
|
[key: string]: string | number | boolean |
||||
|
} |
||||
|
plus: boolean |
||||
|
entities?: { |
||||
|
[key: string]: Table |
||||
|
} |
||||
|
} |
||||
Loading…
Reference in new issue