feat: add DataSources to project json storage (#6160)

* feat: add DataSources project json storage

* refactor: flip to stroage by default with skipFromStorage

* refactor: change datasources storage to be an array

* docs: *

* refactor: remove custom load

* Allow to clear data sources

* Update serialization.ts

* Update tests

---------

Co-authored-by: Artur Arseniev <artur.catch@hotmail.it>

docs/api/data_source_manager.md

@@ -126,6 +126,22 @@ const [dataSource, dataRecord, propPath] = dsm.fromPath('my_data_source_id.recor
 Returns **[DataSource?, DataRecord?, [String][7]?]** An array containing the data source,
 data record, and optional property path.
 
+## store
+
+Store data sources to a JSON object.
+
+Returns **[Object][6]** Stored data sources.
+
+## load
+
+Load data sources from a JSON object.
+
+### Parameters
+
+*   `data` **[Object][6]** The data object containing data sources.
+
+Returns **[Object][6]** Loaded data sources.
+
 [1]: #add
 
 [2]: #get

docs/modules/DataSources.md

@@ -159,6 +159,58 @@ const testDataSource = {
 
 In this example, the `onRecordSetValue` transformer ensures that the `content` property is always an uppercase string.
 
+## Storing DataSources in Project JSON
+
+GrapesJS allows you to control whether a DataSource should be stored statically in the project JSON. This is useful for managing persistent data across project saves and loads.
+
+### Using the `skipFromStorage` Key
+
+When creating a DataSource, you can use the `skipFromStorage` key to specify whether it should be included in the project JSON.
+
+**Example: Creating a DataSource with `skipFromStorage`**
+
+```ts
+const persistentDataSource = {
+  id: 'persistent-datasource',
+  records: [
+    { id: 'id1', content: 'This data will be saved' },
+    { id: 'id2', color: 'blue' },
+  ],
+};
+
+editor.DataSources.add(persistentDataSource);
+
+const temporaryDataSource = {
+  id: 'temporary-datasource',
+  records: [
+    { id: 'id1', content: 'This data will not be saved' },
+  ],
+  skipFromStorage: true,
+};
+
+editor.DataSources.add(temporaryDataSource);
+```
+
+In this example, `persistentDataSource` will be included in the project JSON when the project is saved, while `temporaryDataSource` will not.
+
+### Benefits of Using `skipFromStorage`
+
+1. **Persistent Configuration**: Store configuration data that should persist across project saves and loads.
+2. **Default Data**: Include default data that should always be available in the project.
+3. **Selective Storage**: Choose which DataSources to include in the project JSON, optimizing storage and load times.
+
+### Accessing Stored DataSources
+
+When a project is loaded, GrapesJS will automatically restore the DataSources that were saved. You can then access and use these DataSources as usual.
+
+```ts
+// After loading a project
+const loadedDataSource = editor.DataSources.get('persistent-datasource');
+console.log(loadedDataSource.getRecord('id1').get('content')); // Outputs: "This data will be saved"
+```
+
+Remember that DataSources with `skipFromStorage: true` will not be available after a project is loaded unless you add them programmatically.
+
 ## Benefits of Using DataSources
 
 DataSources are integrated with GrapesJS's runtime and BackboneJS models, enabling dynamic updates and synchronization between your data and UI components. This allows you to:
@@ -207,4 +259,4 @@ In this example, a counter is dynamically updated and displayed in the UI, demon
 1. Injecting configuration
 2. Managing global themes
 3. Mocking & testing
-4. Third-party integrations
+4. Third-party integrations

packages/core/src/data_sources/index.ts

@@ -46,7 +46,7 @@ import { DataSourcesEvents, DataSourceProps } from './types';
 import { Events } from 'backbone';
 
 export default class DataSourceManager extends ItemManagerModule<ModuleConfig, DataSources> {
-  storageKey = '';
+  storageKey = 'dataSources';
   events = DataSourcesEvents;
   destroy(): void {}
 
@@ -148,4 +148,34 @@ export default class DataSourceManager extends ItemManagerModule<ModuleConfig, D
 
     return result;
   }
+
+  /**
+   * Store data sources to a JSON object.
+   * @returns {Array} Stored data sources.
+   */
+  store() {
+    const data: any[] = [];
+    this.all.forEach((dataSource) => {
+      const skipFromStorage = dataSource.get('skipFromStorage');
+      if (!skipFromStorage) {
+        data.push({
+          id: dataSource.id,
+          name: dataSource.get('name' as any),
+          records: dataSource.records.toJSON(),
+          skipFromStorage,
+        });
+      }
+    });
+
+    return { [this.storageKey]: data };
+  }
+
+  /**
+   * Load data sources from a JSON object.
+   * @param {Object} data The data object containing data sources.
+   * @returns {Object} Loaded data sources.
+   */
+  load(data: any) {
+    return this.loadProjectData(data);
+  }
 }

packages/core/src/data_sources/types.ts

@@ -28,8 +28,12 @@ export interface DataSourceProps {
   /**
    * DataSource validation and transformation factories.
    */
-
   transformers?: DataSourceTransformers;
+
+  /**
+   * If true will store the data source in the GrapesJS project.json file.
+   */
+  skipFromStorage?: boolean;
 }
 
 export interface DataSourceTransformers {

packages/core/src/editor/model/Editor.ts

@@ -73,6 +73,7 @@ const storableDeps: (new (em: EditorModel) => IModule & IStorableModule)[] = [
   CssComposer,
   PageManager,
   ComponentManager,
+  DataSourceManager,
 ];
 
 Extender({ $ });

packages/core/test/common.ts

@@ -54,3 +54,35 @@ export function waitEditorEvent(em: Editor | EditorModel, event: string) {
 export function flattenHTML(html: string) {
   return html.replace(/>\s+|\s+</g, (m) => m.trim());
 }
+
+// Filter out the unique ids and selectors replaced with 'data-variable-id'
+// Makes the snapshot more stable
+export function filterObjectForSnapshot(obj: any, parentKey: string = ''): any {
+  const result: any = {};
+
+  for (const key in obj) {
+    if (key === 'id') {
+      result[key] = 'data-variable-id';
+      continue;
+    }
+
+    if (key === 'selectors') {
+      result[key] = obj[key].map(() => 'data-variable-id');
+      continue;
+    }
+
+    if (typeof obj[key] === 'object' && obj[key] !== null) {
+      if (Array.isArray(obj[key])) {
+        result[key] = obj[key].map((item: any) =>
+          typeof item === 'object' ? filterObjectForSnapshot(item, key) : item,
+        );
+      } else {
+        result[key] = filterObjectForSnapshot(obj[key], key);
+      }
+    } else {
+      result[key] = obj[key];
+    }
+  }
+
+  return result;
+}

packages/core/test/specs/data_sources/__snapshots__/serialization.ts.snap

@@ -3,6 +3,7 @@
 exports[`DataSource Serialization .getProjectData ComponentDataVariable 1`] = `
 {
   "assets": [],
+  "dataSources": [],
   "pages": [
     {
       "frames": [
@@ -53,6 +54,7 @@ exports[`DataSource Serialization .getProjectData ComponentDataVariable 1`] = `
 exports[`DataSource Serialization .getProjectData StyleDataVariable 1`] = `
 {
   "assets": [],
+  "dataSources": [],
   "pages": [
     {
       "frames": [
@@ -113,6 +115,7 @@ exports[`DataSource Serialization .getProjectData StyleDataVariable 1`] = `
 exports[`DataSource Serialization .getProjectData TraitDataVariable 1`] = `
 {
   "assets": [],
+  "dataSources": [],
   "pages": [
     {
       "frames": [

packages/core/test/specs/data_sources/__snapshots__/storage.ts.snap

@@ -0,0 +1,123 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`DataSource Storage .getProjectData ComponentDataVariable 1`] = `
+{
+  "assets": [],
+  "dataSources": [
+    {
+      "id": "data-variable-id",
+      "records": [
+        {
+          "content": "Hello World",
+          "id": "data-variable-id",
+        },
+      ],
+    },
+  ],
+  "pages": [
+    {
+      "frames": [
+        {
+          "component": {
+            "components": [
+              {
+                "components": [
+                  {
+                    "defaultValue": "default",
+                    "path": "component-storage.id1.content",
+                    "type": "data-variable",
+                  },
+                ],
+                "tagName": "h1",
+                "type": "text",
+              },
+            ],
+            "docEl": {
+              "tagName": "html",
+            },
+            "head": {
+              "type": "head",
+            },
+            "stylable": [
+              "background",
+              "background-color",
+              "background-image",
+              "background-repeat",
+              "background-attachment",
+              "background-position",
+              "background-size",
+            ],
+            "type": "wrapper",
+          },
+          "id": "data-variable-id",
+        },
+      ],
+      "id": "data-variable-id",
+      "type": "main",
+    },
+  ],
+  "styles": [],
+  "symbols": [],
+}
+`;
+
+exports[`DataSource Storage .loadProjectData ComponentDataVariable 1`] = `
+{
+  "assets": [],
+  "dataSources": [
+    {
+      "id": "data-variable-id",
+      "records": [
+        {
+          "content": "Hello World Updated",
+          "id": "data-variable-id",
+        },
+      ],
+    },
+  ],
+  "pages": [
+    {
+      "frames": [
+        {
+          "component": {
+            "components": [
+              {
+                "components": [
+                  {
+                    "defaultValue": "default",
+                    "path": "component-storage.id1.content",
+                    "type": "data-variable",
+                  },
+                ],
+                "tagName": "h1",
+                "type": "text",
+              },
+            ],
+            "docEl": {
+              "tagName": "html",
+            },
+            "head": {
+              "type": "head",
+            },
+            "stylable": [
+              "background",
+              "background-color",
+              "background-image",
+              "background-repeat",
+              "background-attachment",
+              "background-position",
+              "background-size",
+            ],
+            "type": "wrapper",
+          },
+          "id": "data-variable-id",
+        },
+      ],
+      "id": "data-variable-id",
+      "type": "main",
+    },
+  ],
+  "styles": [],
+  "symbols": [],
+}
+`;

packages/core/test/specs/data_sources/model/TraitDataVariable.ts

@@ -200,7 +200,6 @@ describe('TraitDataVariable', () => {
       dsm.add(inputDataSource);
 
       const cmp = cmpRoot.append({
-        type: 'checkbox',
         tagName: 'input',
         attributes: { type: 'checkbox', name: 'my-checkbox' },
         traits: [

packages/core/test/specs/data_sources/serialization.ts

@@ -5,45 +5,12 @@ import { DataVariableType } from '../../../src/data_sources/model/DataVariable';
 import EditorModel from '../../../src/editor/model/Editor';
 import { ProjectData } from '../../../src/storage_manager';
 import { DataSourceProps } from '../../../src/data_sources/types';
-import { setupTestEditor } from '../../common';
+import { filterObjectForSnapshot, setupTestEditor } from '../../common';
-
-// Filter out the unique ids and selectors replaced with 'data-variable-id'
-// Makes the snapshot more stable
-function filterObjectForSnapshot(obj: any, parentKey: string = ''): any {
-  const result: any = {};
-
-  for (const key in obj) {
-    if (key === 'id') {
-      result[key] = 'data-variable-id';
-      continue;
-    }
-
-    if (key === 'selectors') {
-      result[key] = obj[key].map(() => 'data-variable-id');
-      continue;
-    }
-
-    if (typeof obj[key] === 'object' && obj[key] !== null) {
-      if (Array.isArray(obj[key])) {
-        result[key] = obj[key].map((item: any) =>
-          typeof item === 'object' ? filterObjectForSnapshot(item, key) : item,
-        );
-      } else {
-        result[key] = filterObjectForSnapshot(obj[key], key);
-      }
-    } else {
-      result[key] = obj[key];
-    }
-  }
-
-  return result;
-}
 
 describe('DataSource Serialization', () => {
   let editor: Editor;
   let em: EditorModel;
   let dsm: DataSourceManager;
-  let fixtures: HTMLElement;
   let cmpRoot: ComponentWrapper;
   const componentDataSource: DataSourceProps = {
     id: 'component-serialization',
@@ -51,18 +18,21 @@ describe('DataSource Serialization', () => {
       { id: 'id1', content: 'Hello World' },
       { id: 'id2', color: 'red' },
     ],
+    skipFromStorage: true,
   };
   const styleDataSource: DataSourceProps = {
     id: 'colors-data',
     records: [{ id: 'id1', color: 'red' }],
+    skipFromStorage: true,
   };
   const traitDataSource: DataSourceProps = {
     id: 'test-input',
     records: [{ id: 'id1', value: 'test-value' }],
+    skipFromStorage: true,
   };
 
   beforeEach(() => {
-    ({ editor, em, dsm, cmpRoot, fixtures } = setupTestEditor());
+    ({ editor, em, dsm, cmpRoot } = setupTestEditor());
 
     dsm.add(componentDataSource);
     dsm.add(styleDataSource);
@@ -234,6 +204,7 @@ describe('DataSource Serialization', () => {
         ],
         styles: [],
         symbols: [],
+        dataSources: [componentDataSource],
       };
 
       editor.loadProjectData(componentProjectData);
@@ -299,6 +270,7 @@ describe('DataSource Serialization', () => {
           },
         ],
         symbols: [],
+        dataSources: [styleDataSource],
       };
 
       editor.loadProjectData(componentProjectData);
@@ -362,6 +334,7 @@ describe('DataSource Serialization', () => {
         ],
         styles: [],
         symbols: [],
+        dataSources: [traitDataSource],
       };
 
       editor.loadProjectData(componentProjectData);

packages/core/test/specs/data_sources/storage.ts

@@ -0,0 +1,143 @@
+import Editor from '../../../src/editor';
+import DataSourceManager from '../../../src/data_sources';
+import ComponentWrapper from '../../../src/dom_components/model/ComponentWrapper';
+import { DataVariableType } from '../../../src/data_sources/model/DataVariable';
+import EditorModel from '../../../src/editor/model/Editor';
+import { DataSourceProps } from '../../../src/data_sources/types';
+import { filterObjectForSnapshot, setupTestEditor } from '../../common';
+import { ProjectData } from '../../../src/storage_manager';
+
+describe('DataSource Storage', () => {
+  let editor: Editor;
+  let em: EditorModel;
+  let dsm: DataSourceManager;
+  let cmpRoot: ComponentWrapper;
+  const storedDataSource: DataSourceProps = {
+    id: 'component-storage',
+    records: [{ id: 'id1', content: 'Hello World' }],
+  };
+
+  const nonStoredDataSource: DataSourceProps = {
+    id: 'component-non-storage',
+    records: [{ id: 'id1', content: 'Hello World' }],
+    skipFromStorage: true,
+  };
+
+  beforeEach(() => {
+    ({ editor, em, dsm, cmpRoot } = setupTestEditor());
+
+    dsm.add(storedDataSource);
+    dsm.add(nonStoredDataSource);
+  });
+
+  afterEach(() => {
+    em.destroy();
+  });
+
+  describe('.getProjectData', () => {
+    test('ComponentDataVariable', () => {
+      const dataVariable = {
+        type: DataVariableType,
+        defaultValue: 'default',
+        path: `${storedDataSource.id}.id1.content`,
+      };
+
+      cmpRoot.append({
+        tagName: 'h1',
+        type: 'text',
+        components: [dataVariable],
+      })[0];
+
+      const projectData = editor.getProjectData();
+      const page = projectData.pages[0];
+      const frame = page.frames[0];
+      const component = frame.component.components[0];
+      expect(component.components[0]).toEqual(dataVariable);
+
+      const snapshot = filterObjectForSnapshot(projectData);
+      expect(snapshot).toMatchSnapshot(``);
+
+      const dataSources = projectData.dataSources;
+      expect(dataSources).toEqual([
+        {
+          id: storedDataSource.id,
+          records: storedDataSource.records,
+        },
+      ]);
+    });
+  });
+
+  describe('.loadProjectData', () => {
+    test('ComponentDataVariable', () => {
+      const componentProjectData: ProjectData = {
+        assets: [],
+        dataSources: [
+          {
+            id: storedDataSource.id,
+            records: storedDataSource.records,
+          },
+        ],
+        pages: [
+          {
+            frames: [
+              {
+                component: {
+                  components: [
+                    {
+                      components: [
+                        {
+                          defaultValue: 'default',
+                          path: `${storedDataSource.id}.id1.content`,
+                          type: 'data-variable',
+                        },
+                      ],
+                      tagName: 'h1',
+                      type: 'text',
+                    },
+                  ],
+                  docEl: {
+                    tagName: 'html',
+                  },
+                  head: {
+                    type: 'head',
+                  },
+                  stylable: [
+                    'background',
+                    'background-color',
+                    'background-image',
+                    'background-repeat',
+                    'background-attachment',
+                    'background-position',
+                    'background-size',
+                  ],
+                  type: 'wrapper',
+                },
+                id: 'frame-id',
+              },
+            ],
+            id: 'page-id',
+            type: 'main',
+          },
+        ],
+        styles: [],
+        symbols: [],
+      };
+
+      editor.loadProjectData(componentProjectData);
+
+      const dataSource = dsm.get(storedDataSource.id);
+      const record = dataSource?.getRecord('id1');
+      expect(record?.get('content')).toBe('Hello World');
+
+      expect(editor.getHtml()).toEqual('<body><h1><div>Hello World</div></h1></body>');
+
+      record?.set('content', 'Hello World Updated');
+
+      expect(editor.getHtml()).toEqual('<body><h1><div>Hello World Updated</div></h1></body>');
+
+      const reloadedProjectData = editor.getProjectData();
+      const snapshot = filterObjectForSnapshot(reloadedProjectData);
+      expect(snapshot).toMatchSnapshot(``);
+    });
+  });
+});