budibase/packages/server/src/utilities/csvParser.js

const csv = require("csvtojson")

const VALIDATORS = {
  string: () => true,
  number: attribute => !isNaN(Number(attribute)),
  datetime: attribute => !isNaN(new Date(attribute).getTime()),
}

const PARSERS = {
  datetime: attribute => new Date(attribute).toISOString(),
}

function parse(path, parsers) {
  const result = csv({ parsers }).fromFile(path)

  const schema = {}

  return new Promise((resolve, reject) => {
    result.on("header", headers => {
      for (let header of headers) {
        schema[header] = {
          type: parsers[header] ? parsers[header].type : "string",
          success: true,
        }
      }
    })
    result.fromFile(path).subscribe(row => {
      // For each CSV row
      // parse all the columns that need parsed
      for (let key in parsers) {
        // if the parsing has already failed for a column
        // skip that column
        if (
          !schema[key] ||
          !schema[key].success ||
          schema[key].type === "omit"
        ) {
          continue
        }

        // get the validator for the column type
        const validator = VALIDATORS[parsers[key].type]

        try {
          schema[key].success = !!validator(row[key])
        } catch (err) {
          schema[key].success = false
        }
      }
    })
    result.on("done", error => {
      if (error) {
        console.error(error)
        reject(error)
      }

      resolve(schema)
    })
  })
}

async function transform({ schema, path }) {
  const colParser = {}

  for (let key in schema) {
    colParser[key] = PARSERS[schema[key].type] || schema[key].type
  }

  try {
    const json = await csv({ colParser }).fromFile(path)
    return json
  } catch (err) {
    console.error(`Error transforming CSV to JSON for data import`, err)
  }
}

module.exports = {
  parse,
  transform,
}