Update client lib data fetch models to support generic pagination

packages/client/src/api/queries.js

@@ -4,7 +4,7 @@ import API from "./api"
 /**
  * Executes a query against an external data connector.
  */
-export const executeQuery = async ({ queryId, parameters }) => {
+export const executeQuery = async ({ queryId, pagination, parameters }) => {
   const query = await fetchQueryDefinition(queryId)
   if (query?.datasourceId == null) {
     notificationStore.actions.error("That query couldn't be found")
@@ -14,6 +14,7 @@ export const executeQuery = async ({ queryId, parameters }) => {
     url: `/api/v2/queries/${queryId}`,
     body: {
       parameters,
+      pagination,
     },
   })
   if (res.error) {

packages/client/src/utils/fetch/DataFetch.js

@@ -12,7 +12,7 @@ import { fetchTableDefinition } from "api"
  * internal table or datasource plus.
  * For other types of datasource, this class is overridden and extended.
  */
-export default class TableFetch {
+export default class DataFetch {
   // Feature flags
   supportsSearch = false
   supportsSort = false
@@ -21,7 +21,6 @@ export default class TableFetch {
   // Config
   options = {
     datasource: null,
-    schema: null,
     limit: 10,
 
     // Search config
@@ -53,23 +52,20 @@ export default class TableFetch {
   /**
    * Constructs a new DataFetch instance.
    * @param opts the fetch options
-   * @param flags the datasource feature flags
    */
-  constructor(opts, flags) {
+  constructor(opts) {
     // Merge options with their default values
     this.options = {
       ...this.options,
       ...opts,
     }
 
-    // Update feature flags
-    this.supportsSearch = flags?.supportsSearch || false
-    this.supportsSort = flags?.supportsSort || false
-    this.supportsPagination = flags?.supportsPagination || false
-
     // Bind all functions to properly scope "this"
     this.getData = this.getData.bind(this)
+    this.getPage = this.getPage.bind(this)
     this.getInitialData = this.getInitialData.bind(this)
+    this.determineFeatureFlags = this.determineFeatureFlags.bind(this)
+    this.enrichSchema = this.enrichSchema.bind(this)
     this.refresh = this.refresh.bind(this)
     this.update = this.update.bind(this)
     this.hasNextPage = this.hasNextPage.bind(this)
@@ -116,11 +112,13 @@ export default class TableFetch {
       return
     }
 
-    // Ensure schema exists and enrich it
-    let schema = this.options.schema
-    if (!schema) {
-      schema = await this.constructor.getSchema(datasource)
-    }
+    // Fetch datasource definition and determine feature flags
+    const definition = await this.constructor.getDefinition(datasource)
+    this.determineFeatureFlags(definition)
+
+    // Fetch and enrich schema
+    let schema = this.constructor.getSchema(datasource, definition)
+    schema = this.enrichSchema(schema)
     if (!schema) {
       return
     }
@@ -142,13 +140,16 @@ export default class TableFetch {
     }
 
     // Update store
-    this.store.update($store => ({ ...$store, schema, query, loading: true }))
+    this.store.update($store => ({
+      ...$store,
+      definition,
+      schema,
+      query,
+      loading: true,
+    }))
 
     // Actually fetch data
     const page = await this.getPage()
-    if (page.info) {
-      console.log("new info", page.info)
-    }
     this.store.update($store => ({
       ...$store,
       loading: false,
@@ -165,7 +166,7 @@ export default class TableFetch {
    */
   async getPage() {
     const { sortColumn, sortOrder, sortType, limit } = this.options
-    const { query } = get(this.store)
+    const { query, pageNumber } = get(this.store)
 
     // Get the actual data
     let { rows, info, hasNextPage, cursor } = await this.getData()
@@ -182,7 +183,8 @@ export default class TableFetch {
 
     // If we don't support pagination, do a client-side limit
     if (!this.supportsPagination) {
-      rows = luceneLimit(rows, limit)
+      rows = rows.slice(pageNumber * limit, (pageNumber + 1) * limit)
+      // rows = luceneLimit(rows, limit)
     }
 
     return {
@@ -207,17 +209,27 @@ export default class TableFetch {
   }
 
   /**
-   * Gets the schema definition for a datasource.
+   * Gets the definition for this datasource.
    * Defaults to fetching a table definition.
-   * @param datasource the datasource definition
-   * @return {object} the schema
+   * @param datasource
+   * @return {object} the definition
    */
-  static async getSchema(datasource) {
+  static async getDefinition(datasource) {
     if (!datasource?.tableId) {
       return null
     }
-    const table = await fetchTableDefinition(datasource.tableId)
-    return this.enrichSchema(table?.schema)
+    return await fetchTableDefinition(datasource.tableId)
+  }
+
+  /**
+   * Gets the schema definition for a datasource.
+   * Defaults to getting the "schema" property of the definition.
+   * @param datasource the datasource
+   * @param definition the datasource definition
+   * @return {object} the schema
+   */
+  static getSchema(datasource, definition) {
+    return definition?.schema
   }
 
   /**
@@ -225,7 +237,7 @@ export default class TableFetch {
    * @param schema the datasource schema
    * @return {object} the enriched datasource schema
    */
-  static enrichSchema(schema) {
+  enrichSchema(schema) {
     if (schema == null) {
       return null
     }
@@ -246,6 +258,17 @@ export default class TableFetch {
     return enrichedSchema
   }
 
+  /**
+   * Determine the feature flag for this datasource definition
+   * @param definition
+   */
+  // eslint-disable-next-line no-unused-vars
+  determineFeatureFlags(definition) {
+    this.supportsSearch = false
+    this.supportsSort = false
+    this.supportsPagination = false
+  }
+
   /**
    * Resets the data set and updates options
    * @param newOptions any new options
@@ -319,6 +342,7 @@ export default class TableFetch {
       ...$store,
       loading: true,
       cursor: nextCursor,
+      pageNumber: $store.pageNumber + 1,
     }))
     const { rows, info, hasNextPage, cursor } = await this.getPage()
 
@@ -326,11 +350,10 @@ export default class TableFetch {
     this.store.update($store => {
       let { cursors, pageNumber } = $store
       if (hasNextPage) {
-        cursors[pageNumber + 2] = cursor
+        cursors[pageNumber + 1] = cursor
       }
       return {
         ...$store,
-        pageNumber: pageNumber + 1,
         rows,
         info,
         cursors,
@@ -354,6 +377,7 @@ export default class TableFetch {
       ...$store,
       loading: true,
       cursor: prevCursor,
+      pageNumber: $store.pageNumber - 1,
     }))
     const { rows, info } = await this.getPage()
 
@@ -361,7 +385,6 @@ export default class TableFetch {
     this.store.update($store => {
       return {
         ...$store,
-        pageNumber: $store.pageNumber - 1,
         rows,
         info,
         loading: false,

packages/client/src/utils/fetch/QueryFetch.js

@@ -1,18 +1,25 @@
 import DataFetch from "./DataFetch.js"
 import { executeQuery, fetchQueryDefinition } from "api"
 import { cloneDeep } from "lodash/fp"
+import { get } from "svelte/store"
 
-export default class ViewFetch extends DataFetch {
-  static async getSchema(datasource) {
+export default class QueryFetch extends DataFetch {
+  determineFeatureFlags(definition) {
+    console.log("pagination config", definition?.pagination)
+    this.supportsPagination =
+      definition?.fields?.pagination?.type != null &&
+      definition?.fields?.pagination?.pageParam != null
+  }
+
+  static async getDefinition(datasource) {
     if (!datasource?._id) {
       return null
     }
-    const definition = await fetchQueryDefinition(datasource._id)
-    return this.enrichSchema(definition?.schema)
+    return await fetchQueryDefinition(datasource._id)
   }
 
   async getData() {
-    const { datasource } = this.options
+    const { datasource, limit } = this.options
 
     // Set the default query params
     let parameters = cloneDeep(datasource?.queryParams || {})
@@ -22,13 +29,21 @@ export default class ViewFetch extends DataFetch {
       }
     }
 
-    const { data, ...rest } = await executeQuery({
-      queryId: datasource?._id,
-      parameters,
-    })
+    // Add pagination to query if supported
+    let queryPayload = { queryId: datasource?._id, parameters }
+    if (this.supportsPagination) {
+      const { cursor, definition, pageNumber } = get(this.store)
+      const { type } = definition.fields.pagination.type
+      const page = type === "page" ? pageNumber : cursor
+      queryPayload.pagination = { page, limit }
+    }
+
+    const { data, pagination, ...rest } = await executeQuery(queryPayload)
     return {
       rows: data || [],
       info: rest,
+      cursor: pagination?.page,
+      hasNextPage: data?.length === limit && limit > 0,
     }
   }
 }

packages/client/src/utils/fetch/RelationshipFetch.js

@@ -1,7 +1,7 @@
 import DataFetch from "./DataFetch.js"
 import { fetchRelationshipData } from "api"
 
-export default class ViewFetch extends DataFetch {
+export default class RelationshipFetch extends DataFetch {
   async getData() {
     const { datasource } = this.options
     const res = await fetchRelationshipData({

packages/client/src/utils/fetch/TableFetch.js

@@ -3,12 +3,10 @@ import DataFetch from "./DataFetch.js"
 import { searchTable } from "api"
 
 export default class TableFetch extends DataFetch {
-  constructor(opts) {
-    super(opts, {
-      supportsSearch: true,
-      supportsSort: true,
-      supportsPagination: true,
-    })
+  determineFeatureFlags() {
+    this.supportsSearch = true
+    this.supportsSort = true
+    this.supportsPagination = true
   }
 
   async getData() {

packages/client/src/utils/fetch/ViewFetch.js

@@ -1,18 +1,17 @@
 import DataFetch from "./DataFetch.js"
-import { fetchTableDefinition, fetchViewData } from "api"
+import { fetchViewData } from "api"
 
 export default class ViewFetch extends DataFetch {
-  static async getSchema(datasource) {
-    if (!datasource?.tableId) {
-      return null
-    }
-    const table = await fetchTableDefinition(datasource.tableId)
-    return this.enrichSchema(table?.views?.[datasource.name]?.schema)
+  static async getSchema(datasource, definition) {
+    const schema = definition?.views?.[datasource.name]?.schema
+    console.log(schema)
+    return schema
   }
 
   async getData() {
     const { datasource } = this.options
     const res = await fetchViewData(datasource)
+    console.log(res)
     return {
       rows: res || [],
     }

packages/client/src/utils/schema.js

@@ -35,14 +35,15 @@ export const fetchDatasourceSchema = async datasource => {
 
   // All normal datasource schema can use their corresponsing implementations
   // in the data fetch classes
-  if (type === "table" || type === "link") {
-    return TableFetch.getSchema(datasource)
-  }
-  if (type === "view") {
-    return ViewFetch.getSchema(datasource)
-  }
-  if (type === "query") {
-    return QueryFetch.getSchema(datasource)
+  const handler = {
+    table: TableFetch,
+    link: TableFetch,
+    view: ViewFetch,
+    query: QueryFetch,
+  }[type]
+  if (handler) {
+    const definition = await handler.getDefinition(datasource)
+    return handler.getSchema(datasource, definition)
   }
 
   return null