Improve single schema cache (parse-community#7214)

* Initial Commit

* fix flaky test

* temporary set ci timeout

* turn off ci check

* fix postgres tests

* fix tests

* node flaky test

* remove improvements

* Update SchemaPerformance.spec.js

* fix tests

* revert ci

* Create Singleton Object

* properly clear cache testing

* Cleanup

* remove fit

* try PushController.spec

* try push test rewrite

* try push enqueue time

* Increase test timeout

* remove pg server creation test

* xit push tests

* more xit

* remove skipped tests

* Fix conflicts

* reduce ci timeout

* fix push tests

* Revert "fix push tests"

This reverts commit 05aba62.

* improve initialization

* fix flaky tests

* xit flaky test

* Update CHANGELOG.md

* enable debug logs

* Update LogsRouter.spec.js

* create initial indexes in series

* lint

* horizontal scaling documentation

* Update Changelog

* change horizontalScaling db option

* Add enableSchemaHooks option

* move enableSchemaHooks to databaseOptions

Author: dplewis

CHANGELOG.md

@@ -89,6 +89,9 @@ ___
 ## Unreleased (Master Branch)
 [Full Changelog](https://github.com/parse-community/parse-server/compare/4.5.0...master)
 ### Breaking Changes
+Leveraging database real-time hooks, schema caching has been drastically improved. These improvements allows for reduced calls to the DB, faster queries and prevention of memory leaks. A breaking change can occur if you are horizontally scaling Parse Server (multiple Parse Server instances connecting to the same DB). Set `databaseOptions: { enableSchemaHooks: true }` parameter in [Parse Server Options](https://parseplatform.org/parse-server/api/master/ParseServerOptions.html) (`enableSingleSchemaCache` and `schemaCacheTTL` have been removed). If you are horizontal scaling instances connected to MongoDB, you must use replica set clusters with WiredTiger, see [ChangeStream](https://docs.mongodb.com/manual/changeStreams/#availability)
+
+The new schema cache uses a singleton object that is stored in-memory. In a horizontally scaled environment, if you update the schema in one instance the DB hooks will update the schema in all other instances. `databaseOptions: { enableSchemaHooks: true }` enables the DB hooks. If you have multiple server instances but `databaseOptions: { enableSchemaHooks: false }`, your schema maybe out of sync in your instances (resyncing will happen if an instance restarts). (Diamond Lewis, SebC) [#7214](https://github.com/parse-community/parse-server/issues/7214)
 - Added file upload restriction. File upload is now only allowed for authenticated users by default for improved security. To allow file upload also for Anonymous Users or Public, set the `fileUpload` parameter in the [Parse Server Options](https://parseplatform.org/parse-server/api/master/ParseServerOptions.html) (dblythy, Manuel Trezza) [#7071](https://github.com/parse-community/parse-server/pull/7071)
 ### Notable Changes
 - Added Parse Server Security Check to report weak security settings (Manuel Trezza, dblythy) [#7247](https://github.com/parse-community/parse-server/issues/7247)

package.json

@@ -27,7 +27,7 @@
     "@parse/push-adapter": "3.4.0",
     "@parse/s3-files-adapter": "1.6.0",
     "@parse/simple-mailgun-adapter": "1.2.0",
-    "apollo-server-express": "2.20.0",
+    "apollo-server-express": "2.21.0",
     "bcryptjs": "2.4.3",
     "body-parser": "1.19.0",
     "commander": "5.1.0",
@@ -88,6 +88,7 @@
     "form-data": "3.0.0",
     "husky": "4.2.5",
     "jasmine": "3.5.0",
+    "jasmine-spec-reporter": "6.0.0",
     "jsdoc": "3.6.3",
     "jsdoc-babel": "0.5.0",
     "lint-staged": "10.2.3",

resources/buildConfigDefinitions.js

@@ -53,6 +53,7 @@ function getENVPrefix(iface) {
     'PasswordPolicyOptions' : 'PARSE_SERVER_PASSWORD_POLICY_',
     'FileUploadOptions' : 'PARSE_SERVER_FILE_UPLOAD_',
     'SecurityOptions': 'PARSE_SERVER_SECURITY_',
+    'DatabaseOptions': 'PARSE_SERVER_DATABASE_'
   }
   if (options[iface.id.name]) {
     return options[iface.id.name]
@@ -168,7 +169,7 @@ function parseDefaultValue(elt, value, t) {
     if (type == 'NumberOrBoolean') {
       literalValue = t.numericLiteral(parsers.numberOrBoolParser('')(value));
     }
-    const literalTypes = ['Object', 'SecurityOptions', 'PagesRoute', 'IdempotencyOptions','FileUploadOptions','CustomPagesOptions', 'PagesCustomUrlsOptions', 'PagesOptions'];
+    const literalTypes = ['Object', 'SecurityOptions', 'PagesRoute', 'IdempotencyOptions','FileUploadOptions','CustomPagesOptions', 'PagesCustomUrlsOptions', 'PagesOptions', 'DatabaseOptions'];
     if (literalTypes.includes(type)) {
       const object = parsers.objectParser(value);
       const props = Object.keys(object).map((key) => {

spec/EnableSingleSchemaCache.spec.js

@@ -8,7 +8,9 @@ const WinstonLoggerAdapter = require('../lib/Adapters/Logger/WinstonLoggerAdapte
 
 const loggerController = new LoggerController(new WinstonLoggerAdapter());
 
-describe('LogsRouter', () => {
+describe_only(() => {
+  return process.env.PARSE_SERVER_LOG_LEVEL !== 'debug';
+})('LogsRouter', () => {
   it('can check valid master key of request', done => {
     // Make mock request
     const request = {

spec/LogsRouter.spec.js

@@ -18,6 +18,7 @@ const fakeClient = {
 describe_only_db('mongo')('MongoStorageAdapter', () => {
   beforeEach(done => {
     new MongoStorageAdapter({ uri: databaseURI }).deleteAllClasses().then(done, fail);
+    Config.get(Parse.applicationId).schemaCache.clear();
   });
 
   it('auto-escapes symbols in auth information', () => {
@@ -314,6 +315,8 @@ describe_only_db('mongo')('MongoStorageAdapter', () => {
     await user.signUp();
 
     const database = Config.get(Parse.applicationId).database;
+    await database.adapter.dropAllIndexes('_User');
+
     const preIndexPlan = await database.find(
       '_User',
       { username: 'bugs' },
@@ -549,5 +552,33 @@ describe_only_db('mongo')('MongoStorageAdapter', () => {
         });
       });
     });
+
+    describe('watch _SCHEMA', () => {
+      it('should change', async done => {
+        const adapter = new MongoStorageAdapter({
+          uri: databaseURI,
+          collectionPrefix: '',
+          mongoOptions: { enableSchemaHooks: true },
+        });
+        await reconfigureServer({ databaseAdapter: adapter });
+        expect(adapter.enableSchemaHooks).toBe(true);
+        spyOn(adapter, '_onchange');
+        const schema = {
+          fields: {
+            array: { type: 'Array' },
+            object: { type: 'Object' },
+            date: { type: 'Date' },
+          },
+        };
+
+        await adapter.createClass('Stuff', schema);
+        const myClassSchema = await adapter.getClass('Stuff');
+        expect(myClassSchema).toBeDefined();
+        setTimeout(() => {
+          expect(adapter._onchange).toHaveBeenCalled();
+          done();
+        }, 5000);
+      });
+    });
   }
 });

spec/MongoStorageAdapter.spec.js

@@ -28,12 +28,20 @@ describe('ParseGraphQLController', () => {
     return graphQLConfigRecord;
   };
 
+<<<<<<< HEAD
   beforeAll(async () => {
     parseServer = await global.reconfigureServer({
       schemaCacheTTL: 100,
     });
     databaseController = parseServer.config.databaseController;
     cacheController = parseServer.config.cacheController;
+=======
+  beforeEach(async () => {
+    if (!parseServer) {
+      parseServer = await global.reconfigureServer();
+      databaseController = parseServer.config.databaseController;
+      cacheController = parseServer.config.cacheController;
+>>>>>>> a02014f... Improve single schema cache (#7214)
 
     const defaultFind = databaseController.find.bind(databaseController);
     databaseController.find = async (className, query, ...args) => {

spec/ParseGraphQLController.spec.js

@@ -9,10 +9,8 @@ describe('ParseGraphQLSchema', () => {
   let parseGraphQLSchema;
   const appId = 'test';
 
-  beforeAll(async () => {
-    parseServer = await global.reconfigureServer({
-      schemaCacheTTL: 100,
-    });
+  beforeEach(async () => {
+    parseServer = await global.reconfigureServer();
     databaseController = parseServer.config.databaseController;
     parseGraphQLController = parseServer.config.parseGraphQLController;
     parseGraphQLSchema = new ParseGraphQLSchema({
@@ -68,7 +66,7 @@ describe('ParseGraphQLSchema', () => {
       const graphQLSubscriptions = parseGraphQLSchema.graphQLSubscriptions;
       const newClassObject = new Parse.Object('NewClass');
       await newClassObject.save();
-      await databaseController.schemaCache.clear();
+      await parseServer.config.schemaCache.clear();
       await new Promise(resolve => setTimeout(resolve, 200));
       await parseGraphQLSchema.load();
       expect(parseClasses).not.toBe(parseGraphQLSchema.parseClasses);
@@ -426,14 +424,14 @@ describe('ParseGraphQLSchema', () => {
         log: defaultLogger,
         appId,
       });
-      await parseGraphQLSchema.databaseController.schemaCache.clear();
+      await parseGraphQLSchema.schemaCache.clear();
       const schema1 = await parseGraphQLSchema.load();
       const types1 = parseGraphQLSchema.graphQLTypes;
       const queries1 = parseGraphQLSchema.graphQLQueries;
       const mutations1 = parseGraphQLSchema.graphQLMutations;
       const user = new Parse.Object('User');
       await user.save();
-      await parseGraphQLSchema.databaseController.schemaCache.clear();
+      await parseGraphQLSchema.schemaCache.clear();
       const schema2 = await parseGraphQLSchema.load();
       const types2 = parseGraphQLSchema.graphQLTypes;
       const queries2 = parseGraphQLSchema.graphQLQueries;
@@ -456,14 +454,14 @@ describe('ParseGraphQLSchema', () => {
       });
       const car1 = new Parse.Object('Car');
       await car1.save();
-      await parseGraphQLSchema.databaseController.schemaCache.clear();
+      await parseGraphQLSchema.schemaCache.clear();
       const schema1 = await parseGraphQLSchema.load();
       const types1 = parseGraphQLSchema.graphQLTypes;
       const queries1 = parseGraphQLSchema.graphQLQueries;
       const mutations1 = parseGraphQLSchema.graphQLMutations;
       const car2 = new Parse.Object('car');
       await car2.save();
-      await parseGraphQLSchema.databaseController.schemaCache.clear();
+      await parseGraphQLSchema.schemaCache.clear();
       const schema2 = await parseGraphQLSchema.load();
       const types2 = parseGraphQLSchema.graphQLTypes;
       const queries2 = parseGraphQLSchema.graphQLQueries;
@@ -486,13 +484,13 @@ describe('ParseGraphQLSchema', () => {
       });
       const car = new Parse.Object('Car');
       await car.save();
-      await parseGraphQLSchema.databaseController.schemaCache.clear();
+      await parseGraphQLSchema.schemaCache.clear();
       const schema1 = await parseGraphQLSchema.load();
       const queries1 = parseGraphQLSchema.graphQLQueries;
       const mutations1 = parseGraphQLSchema.graphQLMutations;
       const cars = new Parse.Object('cars');
       await cars.save();
-      await parseGraphQLSchema.databaseController.schemaCache.clear();
+      await parseGraphQLSchema.schemaCache.clear();
       const schema2 = await parseGraphQLSchema.load();
       const queries2 = parseGraphQLSchema.graphQLQueries;
       const mutations2 = parseGraphQLSchema.graphQLMutations;
@@ -532,7 +530,7 @@ describe('ParseGraphQLSchema', () => {
 
       await data.save();
 
-      await parseGraphQLSchema.databaseController.schemaCache.clear();
+      await parseGraphQLSchema.schemaCache.clear();
       await parseGraphQLSchema.load();
 
       const queries1 = parseGraphQLSchema.graphQLQueries;
@@ -569,7 +567,7 @@ describe('ParseGraphQLSchema', () => {
 
       await data.save();
 
-      await parseGraphQLSchema.databaseController.schemaCache.clear();
+      await parseGraphQLSchema.schemaCache.clear();
       await parseGraphQLSchema.load();
 
       const mutations = parseGraphQLSchema.graphQLMutations;