feat: Add option schemaCacheTtl for schema cache pulling as alternative to enableSchemaHooks

spec/SchemaPerformance.spec.js

@@ -204,4 +204,58 @@ describe('Schema Performance', function () {
     );
     expect(getAllSpy.calls.count()).toBe(2);
   });
+
+  it('does reload with schemaCacheTTL', async () => {
+    const databaseURI =
+      process.env.PARSE_SERVER_TEST_DB === 'postgres'
+        ? process.env.PARSE_SERVER_TEST_DATABASE_URI
+        : 'mongodb://localhost:27017/parseServerMongoAdapterTestDatabase';
+    await reconfigureServer({
+      databaseAdapter: undefined,
+      databaseURI,
+      silent: false,
+      databaseOptions: { schemaCacheTTL: 1000 },
+    });
+    const SchemaController = require('../lib/Controllers/SchemaController').SchemaController;
+    const spy = spyOn(SchemaController.prototype, 'reloadData').and.callThrough();
+    Object.defineProperty(spy, 'reloadCalls', {
+      get: () => spy.calls.all().filter(call => call.args[0].clearCache).length,
+    });
+
+    const object = new TestObject();
+    object.set('foo', 'bar');
+    await object.save();
+
+    spy.calls.reset();
+
+    object.set('foo', 'bar');
+    await object.save();
+
+    expect(spy.reloadCalls).toBe(0);
+
+    await new Promise(resolve => setTimeout(resolve, 1100));
+
+    object.set('foo', 'bar');
+    await object.save();
+
+    expect(spy.reloadCalls).toBe(1);
+  });
+
+  it('cannot set invalid databaseOptions', async () => {
+    const expectError = async (key, value, expected) =>
+      expectAsync(
+        reconfigureServer({ databaseAdapter: undefined, databaseOptions: { [key]: value } })
+      ).toBeRejectedWith(`databaseOptions.${key} must be a ${expected}`);
+    for (const databaseOptions of [[], 0, 'string']) {
+      await expectAsync(
+        reconfigureServer({ databaseAdapter: undefined, databaseOptions })
+      ).toBeRejectedWith(`databaseOptions must be an object`);
+    }
+    for (const value of [null, 0, 'string', {}, []]) {
+      await expectError('enableSchemaHooks', value, 'boolean');
+    }
+    for (const value of [null, false, 'string', {}, []]) {
+      await expectError('schemaCacheTTL', value, 'number');
+    }
+  });
 });

src/Adapters/Storage/Mongo/MongoStorageAdapter.js

@@ -139,11 +139,12 @@ export class MongoStorageAdapter implements StorageAdapter {
   _maxTimeMS: ?number;
   canSortOnJoinTables: boolean;
   enableSchemaHooks: boolean;
+  schemaCacheTTL: ?number;
 
   constructor({ uri = defaults.DefaultMongoURI, collectionPrefix = '', mongoOptions = {} }: any) {
     this._uri = uri;
     this._collectionPrefix = collectionPrefix;
-    this._mongoOptions = mongoOptions;
+    this._mongoOptions = { ...mongoOptions };
     this._mongoOptions.useNewUrlParser = true;
     this._mongoOptions.useUnifiedTopology = true;
     this._onchange = () => {};
@@ -152,8 +153,11 @@ export class MongoStorageAdapter implements StorageAdapter {
     this._maxTimeMS = mongoOptions.maxTimeMS;
     this.canSortOnJoinTables = true;
     this.enableSchemaHooks = !!mongoOptions.enableSchemaHooks;
-    delete mongoOptions.enableSchemaHooks;
-    delete mongoOptions.maxTimeMS;
+    this.schemaCacheTTL = mongoOptions.schemaCacheTTL;
+    for (const key of ['enableSchemaHooks', 'schemaCacheTTL', 'maxTimeMS']) {
+      delete mongoOptions[key];
+      delete this._mongoOptions[key];
+    }
   }
 
   watch(callback: () => void): void {

src/Adapters/Storage/Postgres/PostgresStorageAdapter.js

@@ -850,13 +850,18 @@ export class PostgresStorageAdapter implements StorageAdapter {
   _pgp: any;
   _stream: any;
   _uuid: any;
+  schemaCacheTTL: ?number;
 
   constructor({ uri, collectionPrefix = '', databaseOptions = {} }: any) {
+    const options = { ...databaseOptions };
     this._collectionPrefix = collectionPrefix;
     this.enableSchemaHooks = !!databaseOptions.enableSchemaHooks;
-    delete databaseOptions.enableSchemaHooks;
+    this.schemaCacheTTL = databaseOptions.schemaCacheTTL;
+    for (const key of ['enableSchemaHooks', 'schemaCacheTTL']) {
+      delete options[key];
+    }
 
-    const { client, pgp } = createClient(uri, databaseOptions);
+    const { client, pgp } = createClient(uri, options);
     this._client = client;
     this._onchange = () => {};
     this._pgp = pgp;

src/Adapters/Storage/StorageAdapter.js

@@ -30,6 +30,8 @@ export type FullQueryOptions = QueryOptions & UpdateQueryOptions;
 
 export interface StorageAdapter {
   canSortOnJoinTables: boolean;
+  schemaCacheTTL: ?number;
+  enableSchemaHooks: boolean;
 
   classExists(className: string): Promise<boolean>;
   setClassLevelPermissions(className: string, clps: any): Promise<void>;

src/Config.js

@@ -9,6 +9,7 @@ import DatabaseController from './Controllers/DatabaseController';
 import { logLevels as validLogLevels } from './Controllers/LoggerController';
 import {
   AccountLockoutOptions,
+  DatabaseOptions,
   FileUploadOptions,
   IdempotencyOptions,
   LogLevels,
@@ -52,23 +53,20 @@ export class Config {
   }
 
   static put(serverConfiguration) {
-    Config.validate(serverConfiguration);
+    Config.validateOptions(serverConfiguration);
+    Config.validateControllers(serverConfiguration);
     AppCache.put(serverConfiguration.appId, serverConfiguration);
     Config.setupPasswordValidator(serverConfiguration.passwordPolicy);
     return serverConfiguration;
   }
 
-  static validate({
-    verifyUserEmails,
-    userController,
-    appName,
+  static validateOptions({
     publicServerURL,
     revokeSessionOnPasswordReset,
     expireInactiveSessions,
     sessionLength,
     defaultLimit,
     maxLimit,
-    emailVerifyTokenValidityDuration,
     accountLockout,
     passwordPolicy,
     masterKeyIps,
@@ -78,7 +76,6 @@ export class Config {
     readOnlyMasterKey,
     allowHeaders,
     idempotencyOptions,
-    emailVerifyTokenReuseIfValid,
     fileUpload,
     pages,
     security,
@@ -88,6 +85,7 @@ export class Config {
     allowExpiredAuthDataToken,
     logLevels,
     rateLimit,
+    databaseOptions,
   }) {
     if (masterKey === readOnlyMasterKey) {
       throw new Error('masterKey and readOnlyMasterKey should be different');
@@ -97,17 +95,6 @@ export class Config {
       throw new Error('masterKey and maintenanceKey should be different');
     }
 
-    const emailAdapter = userController.adapter;
-    if (verifyUserEmails) {
-      this.validateEmailConfiguration({
-        emailAdapter,
-        appName,
-        publicServerURL,
-        emailVerifyTokenValidityDuration,
-        emailVerifyTokenReuseIfValid,
-      });
-    }
-
     this.validateAccountLockoutPolicy(accountLockout);
     this.validatePasswordPolicy(passwordPolicy);
     this.validateFileUploadOptions(fileUpload);
@@ -136,6 +123,27 @@ export class Config {
     this.validateRequestKeywordDenylist(requestKeywordDenylist);
     this.validateRateLimit(rateLimit);
     this.validateLogLevels(logLevels);
+    this.validateDatabaseOptions(databaseOptions);
+  }
+
+  static validateControllers({
+    verifyUserEmails,
+    userController,
+    appName,
+    publicServerURL,
+    emailVerifyTokenValidityDuration,
+    emailVerifyTokenReuseIfValid,
+  }) {
+    const emailAdapter = userController.adapter;
+    if (verifyUserEmails) {
+      this.validateEmailConfiguration({
+        emailAdapter,
+        appName,
+        publicServerURL,
+        emailVerifyTokenValidityDuration,
+        emailVerifyTokenReuseIfValid,
+      });
+    }
   }
 
   static validateRequestKeywordDenylist(requestKeywordDenylist) {
@@ -526,6 +534,25 @@ export class Config {
     }
   }
 
+  static validateDatabaseOptions(databaseOptions) {
+    if (databaseOptions == undefined) {
+      return;
+    }
+    if (Object.prototype.toString.call(databaseOptions) !== '[object Object]') {
+      throw `databaseOptions must be an object`;
+    }
+    if (databaseOptions.enableSchemaHooks === undefined) {
+      databaseOptions.enableSchemaHooks = DatabaseOptions.enableSchemaHooks.default;
+    } else if (typeof databaseOptions.enableSchemaHooks !== 'boolean') {
+      throw `databaseOptions.enableSchemaHooks must be a boolean`;
+    }
+    if (databaseOptions.schemaCacheTTL === undefined) {
+      databaseOptions.schemaCacheTTL = DatabaseOptions.schemaCacheTTL.default;
+    } else if (typeof databaseOptions.schemaCacheTTL !== 'number') {
+      throw `databaseOptions.schemaCacheTTL must be a number`;
+    }
+  }
+
   static validateRateLimit(rateLimit) {
     if (!rateLimit) {
       return;

src/Controllers/SchemaController.js

@@ -682,6 +682,10 @@ const typeToString = (type: SchemaField | string): string => {
   }
   return `${type.type}`;
 };
+const ttl = {
+  date: Date.now(),
+  duration: undefined,
+};
 
 // Stores the entire schema of the app in a weird hybrid format somewhere between
 // the mongo format and the Parse format. Soon, this will all be Parse format.
@@ -694,10 +698,11 @@ export default class SchemaController {
 
   constructor(databaseAdapter: StorageAdapter) {
     this._dbAdapter = databaseAdapter;
+    const config = Config.get(Parse.applicationId);
     this.schemaData = new SchemaData(SchemaCache.all(), this.protectedFields);
-    this.protectedFields = Config.get(Parse.applicationId).protectedFields;
+    this.protectedFields = config.protectedFields;
 
-    const customIds = Config.get(Parse.applicationId).allowCustomObjectId;
+    const customIds = config.allowCustomObjectId;
 
     const customIdRegEx = /^.{1,}$/u; // 1+ chars
     const autoIdRegEx = /^[a-zA-Z0-9]{1,}$/;
@@ -709,6 +714,21 @@ export default class SchemaController {
     });
   }
 
+  async reloadDataIfNeeded() {
+    if (this._dbAdapter.enableSchemaHooks) {
+      return;
+    }
+    const { date, duration } = ttl || {};
+    if (!duration) {
+      return;
+    }
+    const now = Date.now();
+    if (now - date > duration) {
+      ttl.date = now;
+      await this.reloadData({ clearCache: true });
+    }
+  }
+
   reloadData(options: LoadSchemaOptions = { clearCache: false }): Promise<any> {
     if (this.reloadDataPromise && !options.clearCache) {
       return this.reloadDataPromise;
@@ -729,10 +749,11 @@ export default class SchemaController {
     return this.reloadDataPromise;
   }
 
-  getAllClasses(options: LoadSchemaOptions = { clearCache: false }): Promise<Array<Schema>> {
+  async getAllClasses(options: LoadSchemaOptions = { clearCache: false }): Promise<Array<Schema>> {
     if (options.clearCache) {
       return this.setAllClasses();
     }
+    await this.reloadDataIfNeeded();
     const cached = SchemaCache.all();
     if (cached && cached.length) {
       return Promise.resolve(cached);
@@ -1440,6 +1461,7 @@ export default class SchemaController {
 // Returns a promise for a new Schema.
 const load = (dbAdapter: StorageAdapter, options: any): Promise<SchemaController> => {
   const schema = new SchemaController(dbAdapter);
+  ttl.duration = dbAdapter.schemaCacheTTL;
   return schema.reloadData(options).then(() => schema);
 };
 

src/Options/Definitions.js

@@ -964,6 +964,12 @@ module.exports.DatabaseOptions = {
     action: parsers.booleanParser,
     default: false,
   },
+  schemaCacheTTL: {
+    env: 'PARSE_SERVER_DATABASE_SCHEMA_CACHE_TTL',
+    help:
+      'The TTL for caching the schema for optimizing read/write operations. You should put a long TTL when your DB is in production',
+    action: parsers.numberParser('schemaCacheTTL'),
+  },
 };
 module.exports.AuthAdapter = {
   enabled: {

src/Options/index.js

@@ -543,6 +543,8 @@ export interface DatabaseOptions {
   /* Enables database real-time hooks to update single schema cache. Set to `true` if using multiple Parse Servers instances connected to the same database. Failing to do so will cause a schema change to not propagate to all instances and re-syncing will only happen when the instances restart. To use this feature with MongoDB, a replica set cluster with [change stream](https://docs.mongodb.com/manual/changeStreams/#availability) support is required.
   :DEFAULT: false */
   enableSchemaHooks: ?boolean;
+  /* The TTL for caching the schema for optimizing read/write operations. You should put a long TTL when your DB is in production */
+  schemaCacheTTL: ?number;
 }
 
 export interface AuthAdapter {

src/ParseServer.js

@@ -71,6 +71,7 @@ class ParseServer {
     Parse.initialize(appId, javascriptKey || 'unused', masterKey);
     Parse.serverURL = serverURL;
 
+    Config.validateOptions(options);
     const allControllers = controllers.getControllers(options);
     options.state = 'initialized';
     this.config = Config.put(Object.assign({}, options, allControllers));