server - first passing tests

packages/client/.babelrc

@@ -0,0 +1,12 @@
+{
+    "presets": ["@babel/preset-env"],
+    "sourceMaps": "inline",
+    "retainLines": true,
+    "plugins": [
+        ["@babel/plugin-transform-runtime",
+          {
+            "regenerator": true
+          }
+        ]
+      ]
+}

packages/client/package.json

@@ -4,11 +4,26 @@
   "description": "Client library for talking to budibase web server",
   "main": "index.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "jest"
   },
   "keywords": [
     "budibase"
   ],
   "author": "Michael Shanks",
-  "license": "MPL-2.0"
+  "license": "MPL-2.0",
+  "devDependencies": {
+    "@babel/cli": "^7.4.4",
+    "@babel/core": "^7.4.5",
+    "@babel/plugin-transform-runtime": "^7.4.4",
+    "@babel/preset-env": "^7.4.5",
+    "babel-jest": "^23.6.0",
+    "babel-plugin-transform-es2015-modules-commonjs": "^6.26.2",
+    "cross-env": "^5.1.4",
+    "jest": "^24.8.0",
+    "regenerator-runtime": "^0.11.1",
+    "rollup": "^1.12.0",
+    "rollup-plugin-commonjs": "^10.0.0",
+    "rollup-plugin-local-resolve": "^1.0.7",
+    "rollup-plugin-node-resolve": "^5.0.0"
+  }
 }

packages/core

@@ -1 +1 @@
-Subproject commit 896f3b6c062fcac68de67e33a07732e1293a3ae2
+Subproject commit db358c7931ef77a2c783863d2d47514182400312

packages/server/app.js

@@ -0,0 +1,15 @@
+const Koa = require('koa');
+const app = new Koa();
+const getMasterAppInternal = require("./utilities/masterAppInternal");
+const router = require("./middleware/routers");
+const bodyParser = require('koa-bodyparser');
+
+module.exports = async (config) => {
+
+    app.keys = config.keys;
+    app.context.master = await getMasterAppInternal(config);
+
+    app.use(router(config, app).routes);
+    app.use(bodyParser());
+    return app.listen();
+};

packages/server/appPackages/master/access_levels.json

@@ -1 +1 @@
-{"levels":[{"name":"owner","permissions":[{"type":"create record","nodeKey":"/applications/1-{id}"},{"type":"update record","nodeKey":"/applications/1-{id}"},{"type":"delete record","nodeKey":"/applications/1-{id}"},{"type":"read record","nodeKey":"/applications/1-{id}"},{"type":"update record","nodeKey":"/applications/1-{id}/users/8-{id}"},{"type":"delete record","nodeKey":"/applications/1-{id}/users/8-{id}"},{"type":"create record","nodeKey":"/applications/1-{id}/users/8-{id}"},{"type":"read record","nodeKey":"/applications/1-{id}/users/8-{id}"},{"type":"create record","nodeKey":"/applications/1-{id}/instances/2-{id}"},{"type":"update record","nodeKey":"/applications/1-{id}/instances/2-{id}"},{"type":"delete record","nodeKey":"/applications/1-{id}/instances/2-{id}"},{"type":"read record","nodeKey":"/applications/1-{id}/instances/2-{id}"},{"type":"create record","nodeKey":"/applications/1-{id}/versions/3-{id}"},{"type":"update record","nodeKey":"/applications/1-{id}/versions/3-{id}"},{"type":"delete record","nodeKey":"/applications/1-{id}/versions/3-{id}"},{"type":"read record","nodeKey":"/applications/1-{id}/versions/3-{id}"},{"type":"read index","nodeKey":"/applications/1-{id}/allinstances"},{"type":"read index","nodeKey":"/applications/1-{id}/activeinstances"},{"type":"read index","nodeKey":"/applications/1-{id}/activeusers"},{"type":"read index","nodeKey":"/applications/1-{id}/all_versions"},{"type":"read index","nodeKey":"/applications/1-{id}/instances/2-{id}/users_on_this_instance"},{"type":"read index","nodeKey":"/applications/1-{id}/versions/3-{id}/instances_for_this_version"},{"type":"read index","nodeKey":"/applications/1-{id}/versions/3-{id}/instances_on_this_version"},{"type":"set user access levels"},{"type":"manage collection"},{"type":"list access levels"},{"type":"list users"},{"type":"write access levels"},{"type":"enable or disable user"},{"type":"create temporary access"},{"type":"set password"},{"type":"create user"},{"type":"write templates"}]}],"version":0}
+{"levels":[{"name":"owner","permissions":[{"type":"create record","nodeKey":"/applications/1-{id}"},{"type":"update record","nodeKey":"/applications/1-{id}"},{"type":"delete record","nodeKey":"/applications/1-{id}"},{"type":"read record","nodeKey":"/applications/1-{id}"},{"type":"update record","nodeKey":"/applications/1-{id}/users/8-{id}"},{"type":"delete record","nodeKey":"/applications/1-{id}/users/8-{id}"},{"type":"create record","nodeKey":"/applications/1-{id}/users/8-{id}"},{"type":"read record","nodeKey":"/applications/1-{id}/users/8-{id}"},{"type":"create record","nodeKey":"/applications/1-{id}/instances/2-{id}"},{"type":"update record","nodeKey":"/applications/1-{id}/instances/2-{id}"},{"type":"delete record","nodeKey":"/applications/1-{id}/instances/2-{id}"},{"type":"read record","nodeKey":"/applications/1-{id}/instances/2-{id}"},{"type":"create record","nodeKey":"/applications/1-{id}/versions/3-{id}"},{"type":"update record","nodeKey":"/applications/1-{id}/versions/3-{id}"},{"type":"delete record","nodeKey":"/applications/1-{id}/versions/3-{id}"},{"type":"read record","nodeKey":"/applications/1-{id}/versions/3-{id}"},{"type":"read index","nodeKey":"/applications/1-{id}/allinstances"},{"type":"read index","nodeKey":"/applications/1-{id}/activeinstances"},{"type":"read index","nodeKey":"/applications/1-{id}/activeusers"},{"type":"read index","nodeKey":"/applications/1-{id}/all_versions"},{"type":"read index","nodeKey":"/applications/1-{id}/instances/2-{id}/users_on_this_instance"},{"type":"read index","nodeKey":"/applications/1-{id}/versions/3-{id}/instances_for_this_version"},{"type":"read index","nodeKey":"/applications/1-{id}/versions/3-{id}/instances_on_this_version"},{"type":"set user access levels"},{"type":"manage collection"},{"type":"list access levels"},{"type":"list users"},{"type":"write access levels"},{"type":"enable or disable user"},{"type":"create temporary access"},{"type":"set password"},{"type":"create user"},{"type":"write templates"},{"type":"create record","nodeKey":"/applications/1-{id}/sessions/16-{id}"},{"type":"update record","nodeKey":"/applications/1-{id}/sessions/16-{id}"},{"type":"delete record","nodeKey":"/applications/1-{id}/sessions/16-{id}"},{"type":"create record","nodeKey":"/sessions/17-{id}"},{"type":"read record","nodeKey":"/applications/1-{id}/sessions/16-{id}"},{"type":"update record","nodeKey":"/sessions/17-{id}"},{"type":"delete record","nodeKey":"/sessions/17-{id}"},{"type":"read record","nodeKey":"/sessions/17-{id}"}]}],"version":0}

packages/server/config.js

@@ -0,0 +1,10 @@
+
+
+module.exports = () => ({
+    datastore: "local",
+    datastoreConfig: {
+        rootPath: "./.data"
+    },
+    keys: ["secret1", "secret2"],
+    port: 4001
+})

packages/server/config.sample.js

@@ -0,0 +1,20 @@
+
+
+module.exports = () => ({
+
+    // the datastore type. should link to a module ...
+    // ../datastores/datastores/<datastore>.js
+    datastore: "local",
+
+    // a config object passed to the datastore.databaseManager
+    datastoreConfig: {
+        rootPath: "./.data"
+    },
+
+    // cookie signing keys,these are secret 
+    keys: ["secret1", "secret1"],
+
+    // port for http server to listen on
+    port: 4001
+
+})

packages/server/index.js

@@ -1,8 +1,4 @@
-const Koa = require('koa');
+const app = require("./app");
-const app = new Koa();
+const config = require("./config");
 
-app.use(async ctx => {
+app(config);
-    ctx.body = 'Hello World';
-});
-  
-app.listen(3000);

packages/server/initialise/createMasterDb.js

@@ -1,7 +1,7 @@
 const {initialiseData, setupDatastore,
     getTemplateApi} = require("budibase-core");
-const {getApisForUser, getDatabaseManager,
+const getDatabaseManager = require("../utilities/databaseManager"); 
-    getApisWithFullAccess} = require("./helpers"); 
+const {getApisForUser, getApisWithFullAccess} = require("../utilities/budibaseApi");  
 const masterDbAppDefinition = require("../appPackages/master/appDefinition.json");
 const masterDbAccessLevels = require("../appPackages/master/access_levels.json");
 

packages/server/jest.config.js.later

@@ -0,0 +1,12 @@
+const Sequencer = require('@jest/test-sequencer').default;
+
+const testOrder = [""]
+
+class CustomSequencer extends Sequencer {
+  sort(tests) {
+    // Test structure information
+    // https://github.com/facebook/jest/blob/6b8b1404a1d9254e7d5d90a8934087a9c9899dab/packages/jest-runner/src/types.ts#L17-L21
+    const copyTests = Array.from(tests);
+    return copyTests.sort((testA, testB) => (testA.path > testB.path ? 1 : -1));
+  }
+}

packages/server/middleware/budibase-routing-middleware.js

@@ -1,34 +0,0 @@
-
-export const budibaseRouting = (options) => {
-
-    return async (ctx, next) => {
-
-        ctx.request.path
-
-    };
-
-};
-
-/* api Routes (all /api/..)
-
-POST executeAction/<name> {}
-POST authenticate {}
-POST authenticateTemporaryAccess {}
-POST createUser {}
-POST enabledUser {}
-POST disableUser {}
-GET users 
-GET accessLevels
-POST accessLevels {}
-POST changeMyPassword {}
-POST setPasswordFromTemporaryCode {}
-POST listItems/index/key {}
-POST aggregates/index/key {}
-POST record/key/to/rec {}
-GET record/key/to/rec
-DELETE record/key/to/rec
-POST appHeirarchy {}
-POST actionsAndTriggers {}
-GET appDefinition
-
-*/

packages/server/middleware/router.js

@@ -1,41 +0,0 @@
-const {getAppApis} = require("budibase-core");
-
-module.exports = (datastoreConfig, datastoreModule, method, path) => {
-
-    const datastore = datastoreModule.getDatastore(
-        datastoreConfig);
-        
-    const bb = getAppApis(
-        datastore
-    )
-
-}
-
-
-
-
-
-
-/* api Routes (all /api/..)
-
-POST executeAction/<name> {}
-POST authenticate {}
-POST authenticateTemporaryAccess {}
-POST createUser {}
-POST enabledUser {}
-POST disableUser {}
-GET users 
-GET accessLevels
-POST accessLevels {}
-POST changeMyPassword {}
-POST setPasswordFromTemporaryCode {}
-POST listItems/index/key {}
-POST aggregates/index/key {}
-POST record/key/to/rec {}
-GET record/key/to/rec
-DELETE record/key/to/rec
-POST appHeirarchy {}
-POST actionsAndTriggers {}
-GET appDefinition
-
-*/

packages/server/middleware/routers.js

@@ -0,0 +1,78 @@
+const Router = require("koa-router");
+const session = require("./session");
+const StatusCodes = require("../utilities/statusCodes");
+module.exports = (config, app) => {
+
+    var router = new Router();
+    router.prefix("/:appname/api");
+
+    router
+    .post("/authenticate", async (ctx, next) => {
+        const user = await ctx.master.authenticate(
+            ctx.session._sessCtx.externalKey,
+            ctx.params.appname,
+            ctx.request.body.username,
+            ctx.request.body.password
+        );
+        if(!user) {
+            ctx.throw(StatusCodes.UNAUTHORIZED, "invalid username or password");
+        } 
+        next();
+    })
+    .post("/setPasswordFromTemporaryCode", async (ctx) => {
+
+    })
+    .post("/executeAction/:actionname", async (ctx) => {
+
+    })
+    .post("/createUser", async (ctx) => {
+
+    })
+    .post("/enableUser", async (ctx) => {
+
+    })
+    .post("/disableUser", async (ctx) => {
+
+    })
+    .get("/users", async (ctx) => {
+
+    })
+    .get("/accessLevels", async (ctx) => {
+
+    })
+    .post("/changeMyPassword", async (ctx) => {
+
+    })
+    .post("/listRecords/:indexkey", async (ctx) => {
+
+    })
+    .post("/aggregated/:indexkey", async (ctx) => {
+
+    })
+    .post("/record/:recordkey", async (ctx) => {
+
+    })
+    .get("/record/:recordkey", async (ctx) => {
+
+    })
+    .del("/record/:recordkey", async (ctx) => {
+
+    })
+    .post("/appHeirarchy", async (ctx) => {
+
+    })
+    .post("/actionsAndTriggers", async (ctx) => {
+
+    })
+    .post("/appDefinition", async (ctx) => {
+
+    });
+
+    router.use(session(config, app));
+
+    return router;
+}
+
+/*
+front end get authenticateTemporaryAccess {}
+*/

packages/server/middleware/session.js

@@ -0,0 +1,22 @@
+const session = require('koa-session');
+
+module.exports = (config, app) => {
+    const sessionConfig = {
+        key: 'budi:sess', /** (string) cookie key (default is koa:sess) */
+        /** (number || 'session') maxAge in ms (default is 1 days) */
+        /** 'session' will result in a cookie that expires when session/browser is closed */
+        /** Warning: If a session cookie is stolen, this cookie will never expire */
+        maxAge: 86400000,
+        autoCommit: true, /** (boolean) automatically commit headers (default true) */
+        overwrite: true, /** (boolean) can overwrite or not (default true) */
+        httpOnly: true, /** (boolean) httpOnly or not (default true) */
+        signed: true, /** (boolean) signed or not (default true) */
+        rolling: false, /** (boolean) Force a session identifier cookie to be set on every response. The expiration is reset to the original maxAge, resetting the expiration countdown. (default is false) */
+        renew: false, /** (boolean) renew session when session is nearly expired, so we can always keep user logged in. (default is false)*/
+      };
+
+    return session(
+        sessionConfig,
+        app
+    );
+}

packages/server/package.json

@@ -13,8 +13,17 @@
   "license": "AGPL-3.0-or-later",
   "dependencies": {
     "argon2": "^0.23.0",
-    "budibase-core": "file:../budibase-core/dist",
+    "budibase-core": "file:../core/dist",
+    "koa": "^2.7.0",
+    "koa-bodyparser": "^4.2.1",
+    "koa-router": "^7.4.0",
+    "koa-session": "^5.12.0",
     "rimraf": "^2.6.3",
     "yargs": "^13.2.4"
+  },
+  "devDependencies": {
+    "@jest/test-sequencer": "^24.8.0",
+    "server-destroy": "^1.0.1",
+    "supertest": "^4.0.2"
   }
 }

packages/server/tests/authenticate.spec.js

@@ -0,0 +1,41 @@
+const request = require('supertest');
+const app = require("./testApp")();
+const statusCodes = require("../utilities/statusCodes");
+
+
+describe("authenticate master", () => {
+    beforeAll(() => {
+        return app.start();
+    })
+
+    afterAll(() => {
+        app.server.destroy();
+    })
+
+    it("should return ok correct username and password supplied", () => {
+
+        app.post("/_master/authenticate", {
+            username: app.masterAuth.username,
+            password: app.masterAuth.password
+        })
+        .expect(statusCodes.OK);        
+
+    });
+
+    it("should return unauthorized if username is incorrect", () => {
+        app.post("/_master/authenticate", {
+            username: "unknownuser",
+            password: app.masterAuth.password
+        })
+        .expect(statusCodes.UNAUTHORIZED); 
+    })
+
+    it("should return unauthorized if password is incorrect", () => {
+        app.post("/_master/authenticate", {
+            username: app.masterAuth.username,
+            password: app.masterAuth.password
+        })
+        .expect(statusCodes.UNAUTHORIZED); 
+    })
+})
+

packages/server/tests/testApp.js

@@ -0,0 +1,65 @@
+const app = require("../app");
+const { promisify } = require("util");
+const rimraf = promisify(require("rimraf"));
+const createMasterDb = require("../initialise/createMasterDb");
+const request = require("supertest");
+const fs = require("fs");
+var enableDestroy = require('server-destroy');
+
+const mkdir = promisify(fs.mkdir);
+const masterOwnerName = "test_master";
+const masterOwnerPassword = "test_master_pass";
+
+const config = {
+    datastore: "local",
+    datastoreConfig: {
+        rootPath: "./tests/.data"
+    },
+    keys: ["secret1", "secret2"],
+    port: 4002
+}
+
+
+module.exports = () => {
+
+    let server;
+
+    return ({
+        start: async () => {
+            await reInitialize();
+            server = await app(config);
+            enableDestroy(server);
+        },
+        config,
+        server:() => server,
+        post: (url, body) => postRequest(server,url,body),
+        masterAuth: {
+            username: masterOwnerName,
+            password: masterOwnerPassword
+        }
+    })
+};
+
+const postRequest = (server, url, body) => 
+    request(app)
+    .post(url)
+    .send(body)
+    .set('Accept', 'application/json');
+
+
+const reInitialize = async () => {
+    try {
+        await rimraf(config.datastoreConfig.rootPath);
+    } catch(_){}
+    
+    await mkdir(config.datastoreConfig.rootPath);
+
+    const datastoreModule = require("../../datastores/datastores/" + config.datastore);
+    await createMasterDb(
+        datastoreModule,
+        config.datastoreConfig,
+        masterOwnerName,
+        masterOwnerPassword 
+    );
+}
+

packages/server/initialise/helpers.js → packages/server/utilities/budibaseApi.js

@@ -1,5 +1,5 @@
 const crypto = require("../nodeCrypto");
-const {getDatabaseManager, getAppApis} = require("budibase-core");
+const {getAppApis} = require("budibase-core");
 
 module.exports.getApisWithFullAccess = async (datastore) => {
     const bb = await getAppApis(
@@ -25,5 +25,17 @@ module.exports.getApisForUser = async (datastore, username, password) => {
     return bb;
 }
 
-module.exports.getDatabaseManager = (datastoreModule, datastoreConfig) => 
+module.exports.getApisForSession = async (session) => {
-    getDatabaseManager(datastoreModule.databaseManager(datastoreConfig));
+
+    const user = JSON.parse(session.user_json);
+
+    const bb = await getAppApis(
+        datastore, 
+        null, null, null,
+        crypto
+    );
+
+    bb.asUser(user);
+
+    return bb;
+}

packages/server/initialise/createInstanceDb.js → packages/server/utilities/createInstanceDb.js

@@ -1,5 +1,5 @@
 const {common, getAppApis} = require("budibase-core");
-const {getDatabaseManager} = require("./helpers"); 
+const {getDatabaseManager} = require("./databaseManager"); 
 
 module.exports = async (productSetId, productId, versionId) => {
     const databaseManager = getDatabaseManager(datastoreModule);

packages/server/utilities/databaseManager.js

@@ -0,0 +1,4 @@
+const {getDatabaseManager} = require("budibase-core");
+
+module.exports = (datastoreModule, datastoreConfig) => 
+    getDatabaseManager(datastoreModule.databaseManager(datastoreConfig));

packages/server/utilities/datastore.js

@@ -0,0 +1,2 @@
+module.exports = (config) => 
+    require(`../../datastores/datastores/${config.datastore}`);

packages/server/utilities/exceptions.js

@@ -0,0 +1,16 @@
+const statusCodes = require("./statusCodes");
+
+const errorWithStatus = (message, statusCode) => {
+    const e = new Error(message);
+    e.statusCode = statusCode;
+    return e;
+}
+
+module.exports.unauthorized = (message) => 
+    errorWithStatus(message, statusCodes.UNAUTHORIZED);
+
+module.exports.forbidden = (message) => 
+    errorWithStatus(message, statusCodes.FORBIDDEN);
+
+module.exports.notfound = (message) => 
+    errorWithStatus(message, statusCodes.NOT_FOUND);

packages/server/utilities/masterAppInternal.js

@@ -0,0 +1,134 @@
+const {getApisWithFullAccess, getApisForSession} = require("./budibaseApi");
+const getDatastore = require("./datastore");
+const getDatabaseManager = require("./databaseManager");
+const {$} = require("budibase-core").common;
+const {keyBy} = require("lodash/fp");
+const {unauthorized} = require("./exceptions");
+
+const isMaster = appname => appname === "_master";
+
+module.exports = async (config) => {
+
+    const datastore = getDatastore(config);
+
+    const databaseManager = getDatabaseManager(
+        datastore,
+        config.datastoreConfig);
+
+
+    const bb = await getApisWithFullAccess(
+        datastore.getDatastore(databaseManager.masterDatastoreConfig)
+    ); 
+
+    let applications;
+    const loadApplications = async () => 
+        applications = $(await bb.indexApi.listItems("/all_applications"), [
+            keyBy("name")
+        ]);
+    await loadApplications();
+
+    const getCustomSessionId = (appname, sessionId) => 
+        isMaster(appname)
+        ? bb.recordApi.customId("mastersession", sessionId)
+        : bb.recordApi.customId("session", sessionId);
+
+    
+    const getApplication = async (name) => {
+        if(applications[name]) 
+            return applications[name];
+
+        await loadApplications();
+
+        if(!applications[name])
+            throw new Error("Appliction " + name + " not found");
+
+        return applications[name];
+    };
+
+    const getSession = async (sessionId, appname) => {
+        const customSessionId = getCustomSessionId(appname, sessionId);
+        if(isMaster(appname)) {
+            return await bb.recordApi.load(`/sessions/${customSessionId}`);
+        } 
+        else {
+            const app = await getApplication(appname);
+            return await bb.recordApi.load(`/applications/${app.id}/sessions/${customSessionId}`);
+        }
+    };
+
+    const deleteSession = async (sessionId, appname) => {
+        const customSessionId = getCustomSessionId(appname, sessionId);
+        if(isMaster(appname)) {
+            return await bb.recordApi.delete(`/sessions/${customSessionId}`);
+        } 
+        else {
+            const app = await getApplication(appname);
+            return await bb.recordApi.delete(`/applications/${app.id}/sessions/${customSessionId}`);
+        }
+    };
+
+    const authenticate = async (sessionId, appname, username, password, instanceName="default") => {
+
+        if(isMaster(appname)) {
+            const authUser = await bb.authApi.authenticate(username, password);
+            if(!authUser) {
+                return null;
+            }
+
+            const session = bb.recordApi.getNew("/sessions", "mastersession");
+            bb.recordApi.setCustomId(session, sessionId);
+            session.user_json = JSON.stringify(authUser);
+            await bb.recordApi.save(session);   
+            return session;
+        }
+
+        const app = await getApplication(appname);
+        
+        const userInMaster = await bb.indexApi.listItems(
+            `/applications/${app.id}/users_by_name`,
+            {name:username}
+        ).find(u => u.name === username);
+        
+        const instance = await bb.recordApi.load(
+            userInMaster.instance.key);
+        
+        const bbInstance = await getApisWithFullAccess(
+            datastore.getDatastore(instance.datastoreconfig));
+
+        const authUser = await bbInstance.authApi.authenticate(username, password);
+
+        if(!authUser) {
+            return null;
+        }
+
+        const session = bb.recordApi.getNew(`/applications/${app.id}/sessions`, "session");
+        bb.recordApi.setCustomId(session, sessionId);
+        session.user_json = JSON.stringify(authUser);
+        session.instanceDatastoreConfig = instance.datastoreconfig;
+        await bb.recordApi.save(session);        
+        return session;
+    };
+
+    const getInstanceApiForSession = async (appname, sessionId) => {
+        if(isMaster(appname)) {
+            const customId = bb.recordApi.customId("session", sessionId);
+            const session = await bb.recordApi.load(`/sessions/${customId}`);
+            return await getApisForSession(session);
+        }
+        else {
+            const app = await getApplication(appname);
+            const customId = bb.recordApi.customId("session", sessionId);
+            const session = await bb.recordApi.load(`/applications/${app.id}/sessions/${customId}`);
+            return await getApisForSession(session);
+        }
+    }
+
+    return ({
+        getApplication,
+        getSession,
+        deleteSession, 
+        authenticate,
+        getInstanceApiForSession
+    });
+
+}

packages/server/utilities/statusCodes.js

@@ -0,0 +1,6 @@
+module.exports = {
+    OK:200,
+    UNAUTHORIZED:401,
+    FORBIDDEN:403,
+    NOT_FOUND:404
+};