mirror of https://github.com/Budibase/budibase.git
Andrew Kingston
6 years ago
34 changed files with 8222 additions and 58 deletions
@ -0,0 +1,189 @@ |
|||
<script> |
|||
import { Heading, Body, Button, Select } from "@budibase/bbui" |
|||
import { notifier } from "builderStore/store/notifications" |
|||
import { FIELDS } from "constants/backend" |
|||
import api from "builderStore/api" |
|||
|
|||
const BYTES_IN_KB = 1000 |
|||
const BYTES_IN_MB = 1000000 |
|||
const FILE_SIZE_LIMIT = BYTES_IN_MB * 1 |
|||
|
|||
export let files = [] |
|||
export let dataImport = { |
|||
valid: true, |
|||
schema: {}, |
|||
} |
|||
|
|||
let parseResult |
|||
|
|||
$: schema = parseResult && parseResult.schema |
|||
$: valid = |
|||
!schema || Object.keys(schema).every(column => schema[column].success) |
|||
$: dataImport = { |
|||
valid, |
|||
schema: buildModelSchema(schema), |
|||
path: files[0] && files[0].path, |
|||
} |
|||
|
|||
function buildModelSchema(schema) { |
|||
const modelSchema = {} |
|||
for (let key in schema) { |
|||
const type = schema[key].type |
|||
|
|||
if (type === "omit") continue |
|||
|
|||
modelSchema[key] = { |
|||
name: key, |
|||
type, |
|||
constraints: FIELDS[type.toUpperCase()].constraints, |
|||
} |
|||
} |
|||
return modelSchema |
|||
} |
|||
|
|||
async function validateCSV() { |
|||
const response = await api.post("/api/models/csv/validate", { |
|||
file: files[0], |
|||
schema: schema || {}, |
|||
}) |
|||
|
|||
parseResult = await response.json() |
|||
|
|||
if (response.status !== 200) { |
|||
notifier.danger("CSV Invalid, please try another CSV file") |
|||
return [] |
|||
} |
|||
} |
|||
|
|||
async function handleFile(evt) { |
|||
const fileArray = Array.from(evt.target.files) |
|||
const filesToProcess = fileArray.map(({ name, path, size }) => ({ |
|||
name, |
|||
path, |
|||
size, |
|||
})) |
|||
|
|||
if (filesToProcess.some(file => file.size >= FILE_SIZE_LIMIT)) { |
|||
notifier.danger( |
|||
`Files cannot exceed ${FILE_SIZE_LIMIT / |
|||
BYTES_IN_MB}MB. Please try again with smaller files.` |
|||
) |
|||
return |
|||
} |
|||
|
|||
files = filesToProcess |
|||
|
|||
await validateCSV() |
|||
} |
|||
|
|||
async function omitColumn(columnName) { |
|||
schema[columnName].type = "omit" |
|||
await validateCSV() |
|||
} |
|||
|
|||
const handleTypeChange = column => evt => { |
|||
schema[column].type = evt.target.value |
|||
validateCSV() |
|||
} |
|||
</script> |
|||
|
|||
<div class="dropzone"> |
|||
<input id="file-upload" accept=".csv" type="file" on:change={handleFile} /> |
|||
<label for="file-upload" class:uploaded={files[0]}> |
|||
{#if files[0]}{files[0].name}{:else}Upload{/if} |
|||
</label> |
|||
</div> |
|||
<div class="schema-fields"> |
|||
{#if schema} |
|||
{#each Object.keys(schema).filter(key => schema[key].type !== 'omit') as columnName} |
|||
<div class="field"> |
|||
<span>{columnName}</span> |
|||
<Select |
|||
secondary |
|||
thin |
|||
bind:value={schema[columnName].type} |
|||
on:change={handleTypeChange(columnName)}> |
|||
<option value={'string'}>Text</option> |
|||
<option value={'number'}>Number</option> |
|||
<option value={'datetime'}>Date</option> |
|||
</Select> |
|||
<span class="field-status" class:error={!schema[columnName].success}> |
|||
{schema[columnName].success ? 'Success' : 'Failure'} |
|||
</span> |
|||
<i |
|||
class="omit-button ri-close-circle-fill" |
|||
on:click={() => omitColumn(columnName)} /> |
|||
</div> |
|||
{/each} |
|||
{/if} |
|||
</div> |
|||
|
|||
<style> |
|||
.dropzone { |
|||
text-align: center; |
|||
display: flex; |
|||
align-items: center; |
|||
flex-direction: column; |
|||
border-radius: 10px; |
|||
transition: all 0.3s; |
|||
} |
|||
|
|||
.field-status { |
|||
color: var(--green); |
|||
justify-self: center; |
|||
font-weight: 500; |
|||
} |
|||
|
|||
.error { |
|||
color: var(--red); |
|||
} |
|||
|
|||
.uploaded { |
|||
color: var(--blue); |
|||
} |
|||
|
|||
input[type="file"] { |
|||
display: none; |
|||
} |
|||
|
|||
label { |
|||
font-family: var(--font-sans); |
|||
cursor: pointer; |
|||
font-weight: 500; |
|||
box-sizing: border-box; |
|||
overflow: hidden; |
|||
border-radius: var(--border-radius-s); |
|||
color: var(--ink); |
|||
padding: var(--spacing-s) var(--spacing-l); |
|||
transition: all 0.2s ease 0s; |
|||
display: inline-flex; |
|||
text-rendering: optimizeLegibility; |
|||
min-width: auto; |
|||
outline: none; |
|||
font-feature-settings: "case" 1, "rlig" 1, "calt" 0; |
|||
-webkit-box-align: center; |
|||
user-select: none; |
|||
flex-shrink: 0; |
|||
align-items: center; |
|||
justify-content: center; |
|||
width: 100%; |
|||
background-color: var(--grey-2); |
|||
font-size: var(--font-size-xs); |
|||
} |
|||
|
|||
.omit-button { |
|||
font-size: 1.2em; |
|||
color: var(--grey-7); |
|||
cursor: pointer; |
|||
justify-self: flex-end; |
|||
} |
|||
|
|||
.field { |
|||
display: grid; |
|||
grid-template-columns: repeat(4, 1fr); |
|||
margin-top: var(--spacing-m); |
|||
align-items: center; |
|||
grid-gap: var(--spacing-m); |
|||
font-size: var(--font-size-xs); |
|||
} |
|||
</style> |
|||
|
After Width: | Height: | Size: 52 KiB |
@ -0,0 +1,73 @@ |
|||
const csv = require("csvtojson") |
|||
|
|||
const VALIDATORS = { |
|||
string: () => true, |
|||
number: attribute => !isNaN(Number(attribute)), |
|||
datetime: attribute => !isNaN(new Date(attribute).getTime()), |
|||
} |
|||
|
|||
const PARSERS = { |
|||
datetime: attribute => new Date(attribute).toISOString(), |
|||
} |
|||
|
|||
function parse(path, parsers) { |
|||
const result = csv().fromFile(path) |
|||
|
|||
const schema = {} |
|||
|
|||
return new Promise((resolve, reject) => { |
|||
result.on("header", headers => { |
|||
for (let header of headers) { |
|||
schema[header] = { |
|||
type: parsers[header] ? parsers[header].type : "string", |
|||
success: true, |
|||
} |
|||
} |
|||
}) |
|||
result.fromFile(path).subscribe(row => { |
|||
// For each CSV row parse all the columns that need parsed
|
|||
for (let key in parsers) { |
|||
if (!schema[key] || schema[key].success) { |
|||
// get the validator for the column type
|
|||
const validator = VALIDATORS[parsers[key].type] |
|||
|
|||
try { |
|||
// allow null/undefined values
|
|||
schema[key].success = !row[key] || validator(row[key]) |
|||
} catch (err) { |
|||
schema[key].success = false |
|||
} |
|||
} |
|||
} |
|||
}) |
|||
result.on("done", error => { |
|||
if (error) { |
|||
console.error(error) |
|||
reject(error) |
|||
} |
|||
|
|||
resolve(schema) |
|||
}) |
|||
}) |
|||
} |
|||
|
|||
async function transform({ schema, path }) { |
|||
const colParser = {} |
|||
|
|||
for (let key in schema) { |
|||
colParser[key] = PARSERS[schema[key].type] || schema[key].type |
|||
} |
|||
|
|||
try { |
|||
const json = await csv({ colParser }).fromFile(path) |
|||
return json |
|||
} catch (err) { |
|||
console.error(`Error transforming CSV to JSON for data import`, err) |
|||
throw err |
|||
} |
|||
} |
|||
|
|||
module.exports = { |
|||
parse, |
|||
transform, |
|||
} |
|||
@ -0,0 +1,21 @@ |
|||
// Jest Snapshot v1, https://goo.gl/fbAQLP |
|||
|
|||
exports[`CSV Parser transformation transforms a CSV file into JSON 1`] = ` |
|||
Array [ |
|||
Object { |
|||
"Address": "5 Sesame Street", |
|||
"Age": 4324, |
|||
"Name": "Bert", |
|||
}, |
|||
Object { |
|||
"Address": "1 World Trade Center", |
|||
"Age": 34, |
|||
"Name": "Ernie", |
|||
}, |
|||
Object { |
|||
"Address": "44 Second Avenue", |
|||
"Age": 23423, |
|||
"Name": "Big Bird", |
|||
}, |
|||
] |
|||
`; |
|||
@ -0,0 +1,108 @@ |
|||
const csvParser = require("../csvParser"); |
|||
|
|||
const CSV_PATH = __dirname + "/test.csv"; |
|||
|
|||
const SCHEMAS = { |
|||
VALID: { |
|||
Age: { |
|||
type: "number", |
|||
}, |
|||
}, |
|||
INVALID: { |
|||
Address: { |
|||
type: "number", |
|||
}, |
|||
Age: { |
|||
type: "number", |
|||
}, |
|||
}, |
|||
IGNORE: { |
|||
Address: { |
|||
type: "omit", |
|||
}, |
|||
Age: { |
|||
type: "omit", |
|||
}, |
|||
}, |
|||
BROKEN: { |
|||
Address: { |
|||
type: "datetime", |
|||
} |
|||
}, |
|||
}; |
|||
|
|||
describe("CSV Parser", () => { |
|||
describe("parsing", () => { |
|||
it("returns status and types for a valid CSV transformation", async () => { |
|||
expect( |
|||
await csvParser.parse(CSV_PATH, SCHEMAS.VALID) |
|||
).toEqual({ |
|||
Address: { |
|||
success: true, |
|||
type: "string", |
|||
}, |
|||
Age: { |
|||
success: true, |
|||
type: "number", |
|||
}, |
|||
Name: { |
|||
success: true, |
|||
type: "string", |
|||
}, |
|||
}); |
|||
}); |
|||
|
|||
it("returns status and types for an invalid CSV transformation", async () => { |
|||
expect( |
|||
await csvParser.parse(CSV_PATH, SCHEMAS.INVALID) |
|||
).toEqual({ |
|||
Address: { |
|||
success: false, |
|||
type: "number", |
|||
}, |
|||
Age: { |
|||
success: true, |
|||
type: "number", |
|||
}, |
|||
Name: { |
|||
success: true, |
|||
type: "string", |
|||
}, |
|||
}); |
|||
}); |
|||
}); |
|||
|
|||
describe("transformation", () => { |
|||
it("transforms a CSV file into JSON", async () => { |
|||
expect( |
|||
await csvParser.transform({ |
|||
schema: SCHEMAS.VALID, |
|||
path: CSV_PATH, |
|||
}) |
|||
).toMatchSnapshot(); |
|||
}); |
|||
|
|||
it("transforms a CSV file into JSON ignoring certain fields", async () => { |
|||
expect( |
|||
await csvParser.transform({ |
|||
schema: SCHEMAS.IGNORE, |
|||
path: CSV_PATH, |
|||
}) |
|||
).toEqual([ |
|||
{ |
|||
Name: "Bert" |
|||
}, |
|||
{ |
|||
Name: "Ernie" |
|||
}, |
|||
{ |
|||
Name: "Big Bird" |
|||
} |
|||
]); |
|||
}); |
|||
|
|||
it("throws an error on invalid schema", async () => { |
|||
await expect(csvParser.transform({ schema: SCHEMAS.BROKEN, path: CSV_PATH })).rejects.toThrow() |
|||
}); |
|||
}); |
|||
}); |
|||
|
File diff suppressed because it is too large
Loading…
Reference in new issue