forked from tsai/budibase
mike12345567
4 years ago
7 changed files with 188 additions and 98 deletions
@ -0,0 +1,63 @@ |
|||
const { |
|||
ObjectStoreBuckets, |
|||
ObjectStore, |
|||
retrieve, |
|||
uploadDirectory, |
|||
makeSureBucketExists, |
|||
} = require("@budibase/backend-core/objectStore") |
|||
const fs = require("fs") |
|||
const { join } = require("path") |
|||
const { TEMP_DIR, MINIO_DIR } = require("./utils") |
|||
const { progressBar } = require("../utils") |
|||
|
|||
const bucketList = Object.values(ObjectStoreBuckets) |
|||
|
|||
exports.exportObjects = async () => { |
|||
const path = join(TEMP_DIR, MINIO_DIR) |
|||
fs.mkdirSync(path) |
|||
let fullList = [] |
|||
for (let bucket of bucketList) { |
|||
const client = ObjectStore(bucket) |
|||
try { |
|||
await client.headBucket().promise() |
|||
} catch (err) { |
|||
continue |
|||
} |
|||
const list = await client.listObjectsV2().promise() |
|||
fullList = fullList.concat(list.Contents.map(el => ({ ...el, bucket }))) |
|||
} |
|||
const bar = progressBar(fullList.length) |
|||
let count = 0 |
|||
for (let object of fullList) { |
|||
const filename = object.Key |
|||
const data = await retrieve(object.bucket, filename) |
|||
const possiblePath = filename.split("/") |
|||
if (possiblePath.length > 1) { |
|||
const dirs = possiblePath.slice(0, possiblePath.length - 1) |
|||
fs.mkdirSync(join(path, object.bucket, ...dirs), { recursive: true }) |
|||
} |
|||
fs.writeFileSync(join(path, object.bucket, ...possiblePath), data) |
|||
bar.update(++count) |
|||
} |
|||
bar.stop() |
|||
} |
|||
|
|||
exports.importObjects = async () => { |
|||
const path = join(TEMP_DIR, MINIO_DIR) |
|||
const buckets = fs.readdirSync(path) |
|||
let total = 0 |
|||
buckets.forEach(bucket => { |
|||
const files = fs.readdirSync(join(path, bucket)) |
|||
total += files.length |
|||
}) |
|||
const bar = progressBar(total) |
|||
let count = 0 |
|||
for (let bucket of buckets) { |
|||
const client = ObjectStore(bucket) |
|||
await makeSureBucketExists(client, bucket) |
|||
const files = await uploadDirectory(bucket, join(path, bucket), "/") |
|||
count += files.length |
|||
bar.update(count) |
|||
} |
|||
bar.stop() |
|||
} |
|||
@ -0,0 +1,92 @@ |
|||
const dotenv = require("dotenv") |
|||
const fs = require("fs") |
|||
const { string } = require("../questions") |
|||
const { env } = require("@budibase/backend-core") |
|||
const { getPouch } = require("@budibase/backend-core/db") |
|||
|
|||
exports.DEFAULT_COUCH = "http://budibase:budibase@localhost:10000/db/" |
|||
exports.DEFAULT_MINIO = "http://localhost:10000/" |
|||
exports.TEMP_DIR = ".temp" |
|||
exports.COUCH_DIR = "couchdb" |
|||
exports.MINIO_DIR = "minio" |
|||
|
|||
const REQUIRED = [ |
|||
{ value: "MAIN_PORT", default: "10000" }, |
|||
{ value: "COUCH_DB_URL", default: exports.DEFAULT_COUCH }, |
|||
{ value: "MINIO_URL", default: exports.DEFAULT_MINIO }, |
|||
{ value: "MINIO_ACCESS_KEY" }, |
|||
{ value: "MINIO_SECRET_KEY" }, |
|||
] |
|||
|
|||
exports.checkURLs = config => { |
|||
const mainPort = config["MAIN_PORT"], |
|||
username = config["COUCH_DB_USER"], |
|||
password = config["COUCH_DB_PASSWORD"] |
|||
if (!config["COUCH_DB_URL"] && mainPort && username && password) { |
|||
config[ |
|||
"COUCH_DB_URL" |
|||
] = `http://${username}:${password}@localhost:${mainPort}/db/` |
|||
} |
|||
if (!config["MINIO_URL"]) { |
|||
config["MINIO_URL"] = exports.DEFAULT_MINIO |
|||
} |
|||
return config |
|||
} |
|||
|
|||
exports.askQuestions = async () => { |
|||
console.log( |
|||
"*** NOTE: use a .env file to load these parameters repeatedly ***" |
|||
) |
|||
let config = {} |
|||
for (let property of REQUIRED) { |
|||
config[property.value] = await string(property.value, property.default) |
|||
} |
|||
return config |
|||
} |
|||
|
|||
exports.loadEnvironment = path => { |
|||
if (!fs.existsSync(path)) { |
|||
throw "Unable to file specified .env file" |
|||
} |
|||
const env = fs.readFileSync(path, "utf8") |
|||
const config = exports.checkURLs(dotenv.parse(env)) |
|||
for (let required of REQUIRED) { |
|||
if (!config[required.value]) { |
|||
throw `Cannot find "${required.value}" property in .env file` |
|||
} |
|||
} |
|||
return config |
|||
} |
|||
|
|||
// true is the default value passed by commander
|
|||
exports.getConfig = async (envFile = true) => { |
|||
let config |
|||
if (envFile !== true) { |
|||
config = exports.loadEnvironment(envFile) |
|||
} else { |
|||
config = await exports.askQuestions() |
|||
} |
|||
for (let required of REQUIRED) { |
|||
env._set(required.value, config[required.value]) |
|||
} |
|||
return config |
|||
} |
|||
|
|||
exports.replication = (from, to) => { |
|||
return new Promise((resolve, reject) => { |
|||
from.replicate |
|||
.to(to) |
|||
.on("complete", () => { |
|||
resolve() |
|||
}) |
|||
.on("error", err => { |
|||
reject(err) |
|||
}) |
|||
}) |
|||
} |
|||
|
|||
exports.getPouches = () => { |
|||
const Remote = getPouch({ replication: true }) |
|||
const Local = getPouch({ onDisk: true, directory: exports.TEMP_DIR }) |
|||
return { Remote, Local } |
|||
} |
|||
Loading…
Reference in new issue