mirror of https://github.com/Budibase/budibase.git
Martin McKeaveney
6 years ago
9 changed files with 301 additions and 83 deletions
@ -0,0 +1,189 @@ |
|||
<script> |
|||
import { Heading, Body, Button, Select } from "@budibase/bbui" |
|||
import { notifier } from "builderStore/store/notifications" |
|||
import { FIELDS } from "constants/backend" |
|||
|
|||
const BYTES_IN_KB = 1000 |
|||
const BYTES_IN_MB = 1000000 |
|||
const FILE_SIZE_LIMIT = BYTES_IN_MB * 1 |
|||
|
|||
export let files = [] |
|||
export let dataImport = { |
|||
valid: true, |
|||
schema: {}, |
|||
} |
|||
|
|||
let parseResult |
|||
|
|||
$: schema = parseResult && parseResult.schema |
|||
$: valid = |
|||
schema && Object.keys(schema).every(column => schema[column].success) |
|||
$: dataImport = { |
|||
valid, |
|||
schema: buildModelSchema(schema), |
|||
path: files.length && files[0].path, |
|||
} |
|||
|
|||
function buildModelSchema(schema) { |
|||
const modelSchema = {} |
|||
for (let key in schema) { |
|||
const type = schema[key].type |
|||
modelSchema[key] = { |
|||
name: key, |
|||
type, |
|||
constraints: FIELDS[type.toUpperCase()].constraints, |
|||
} |
|||
} |
|||
return modelSchema |
|||
} |
|||
|
|||
async function validateCSV() { |
|||
const response = await fetch("/api/models/csv/validate", { |
|||
method: "POST", |
|||
body: JSON.stringify({ |
|||
file: files[0], |
|||
schema: schema || {}, |
|||
}), |
|||
headers: { |
|||
"Content-Type": "application/json", |
|||
Accept: "application/json", |
|||
}, |
|||
}) |
|||
|
|||
parseResult = await response.json() |
|||
|
|||
if (response.status !== 200) { |
|||
notifier.danger("CSV Invalid, please try another CSV file") |
|||
return [] |
|||
} |
|||
} |
|||
|
|||
async function handleFile(evt) { |
|||
const fileArray = Array.from(evt.target.files) |
|||
const filesToProcess = fileArray.map(({ name, path, size }) => ({ |
|||
name, |
|||
path, |
|||
size, |
|||
})) |
|||
|
|||
if (filesToProcess.some(file => file.size >= FILE_SIZE_LIMIT)) { |
|||
notifier.danger( |
|||
`Files cannot exceed ${FILE_SIZE_LIMIT / |
|||
BYTES_IN_MB}MB. Please try again with smaller files.` |
|||
) |
|||
return |
|||
} |
|||
|
|||
files = filesToProcess |
|||
|
|||
await validateCSV() |
|||
} |
|||
|
|||
function omitColumn(columnName) { |
|||
parsers[columnName] = PARSERS.omit |
|||
} |
|||
|
|||
const handleTypeChange = column => evt => { |
|||
schema[column].type = evt.target.value |
|||
validateCSV() |
|||
} |
|||
</script> |
|||
|
|||
<div class="dropzone"> |
|||
<input id="file-upload" accept=".csv" type="file" on:change={handleFile} /> |
|||
<label for="file-upload"> |
|||
{#if files[0]}{files[0].name}{:else}Upload{/if} |
|||
</label> |
|||
</div> |
|||
<div class="schema-fields"> |
|||
{#if schema} |
|||
{#each Object.keys(schema) as columnName} |
|||
<div class="field"> |
|||
<span>{columnName}</span> |
|||
<Select |
|||
secondary |
|||
thin |
|||
bind:value={schema[columnName].type} |
|||
on:change={handleTypeChange(columnName)}> |
|||
<option value={'string'}>Text</option> |
|||
<option value={'number'}>Number</option> |
|||
<option value={'datetime'}>Date</option> |
|||
</Select> |
|||
<span |
|||
class:success={schema[columnName].success} |
|||
class:error={!schema[columnName].success}> |
|||
{schema[columnName].success ? 'Success' : 'Failure'} |
|||
</span> |
|||
<i |
|||
class="omit-button ri-close-circle-fill" |
|||
on:click={() => omitColumn(columnName)} /> |
|||
</div> |
|||
{/each} |
|||
{/if} |
|||
</div> |
|||
|
|||
<style> |
|||
.dropzone { |
|||
text-align: center; |
|||
display: flex; |
|||
align-items: center; |
|||
flex-direction: column; |
|||
border-radius: 10px; |
|||
transition: all 0.3s; |
|||
} |
|||
|
|||
.success { |
|||
color: green; |
|||
} |
|||
|
|||
.error { |
|||
color: red; |
|||
} |
|||
|
|||
input[type="file"] { |
|||
display: none; |
|||
} |
|||
|
|||
label { |
|||
font-family: var(--font-sans); |
|||
cursor: pointer; |
|||
font-weight: 600; |
|||
box-sizing: border-box; |
|||
overflow: hidden; |
|||
border-radius: var(--border-radius-s); |
|||
color: var(--white); |
|||
padding: var(--spacing-s) var(--spacing-l); |
|||
transition: all 0.2s ease 0s; |
|||
display: inline-flex; |
|||
text-rendering: optimizeLegibility; |
|||
min-width: auto; |
|||
outline: none; |
|||
font-feature-settings: "case" 1, "rlig" 1, "calt" 0; |
|||
-webkit-box-align: center; |
|||
user-select: none; |
|||
flex-shrink: 0; |
|||
align-items: center; |
|||
justify-content: center; |
|||
margin-top: 10px; |
|||
width: 100%; |
|||
border: solid 1.5px var(--ink); |
|||
background-color: var(--ink); |
|||
} |
|||
|
|||
/* .schema-fields { |
|||
} */ |
|||
|
|||
.omit-button { |
|||
font-size: 1.2em; |
|||
color: var(--grey-7); |
|||
cursor: pointer; |
|||
} |
|||
|
|||
.field { |
|||
display: grid; |
|||
grid-template-columns: repeat(4, 1fr); |
|||
margin-top: var(--spacing-m); |
|||
align-items: center; |
|||
grid-gap: var(--spacing-m); |
|||
} |
|||
</style> |
|||
|
After Width: | Height: | Size: 52 KiB |
@ -0,0 +1,76 @@ |
|||
const csv = require("csvtojson") |
|||
|
|||
const VALIDATORS = { |
|||
string: () => true, |
|||
number: attribute => !isNaN(Number(attribute)), |
|||
datetime: attribute => !isNaN(new Date(attribute).getTime()), |
|||
} |
|||
|
|||
const PARSERS = { |
|||
string: attribute => attribute.toString(), |
|||
number: attribute => Number(attribute), |
|||
datetime: attribute => new Date(attribute).toISOString(), |
|||
} |
|||
|
|||
function parse(path, parsers) { |
|||
const result = csv().fromFile(path) |
|||
|
|||
const schema = {} |
|||
|
|||
return new Promise((resolve, reject) => { |
|||
result.on("header", headers => { |
|||
for (let header of headers) { |
|||
schema[header] = { |
|||
type: parsers[header] ? parsers[header].type : "string", |
|||
success: true, |
|||
} |
|||
} |
|||
}) |
|||
result.fromFile(path).subscribe(row => { |
|||
// For each CSV row
|
|||
// parse all the columns that need parsed
|
|||
for (let key in parsers) { |
|||
// if the schema has already borked for a parser, skip this column
|
|||
if (!schema[key] || !schema[key].success) continue |
|||
|
|||
// get the validator
|
|||
const validator = VALIDATORS[parsers[key].type] |
|||
|
|||
try { |
|||
schema[key].success = !!validator(row[key]) |
|||
} catch (err) { |
|||
schema[key].success = false |
|||
} |
|||
} |
|||
}) |
|||
result.on("done", error => { |
|||
if (error) { |
|||
console.error(error) |
|||
reject(error) |
|||
} |
|||
|
|||
resolve(schema) |
|||
}) |
|||
}) |
|||
} |
|||
|
|||
// TODO: significant refactor
|
|||
async function transform({ schema, path }) { |
|||
const colParser = {} |
|||
|
|||
for (let key in schema) { |
|||
colParser[key] = PARSERS[schema[key].type] |
|||
} |
|||
|
|||
try { |
|||
const json = await csv({ colParser }).fromFile(path) |
|||
return json |
|||
} catch (err) { |
|||
console.error(`Error transforming CSV to JSON for data import`, err) |
|||
} |
|||
} |
|||
|
|||
module.exports = { |
|||
parse, |
|||
transform, |
|||
} |
|||
Loading…
Reference in new issue