Add idempotency

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ### master
 [Full Changelog](https://github.com/parse-community/parse-server/compare/4.2.0...master)
+- NEW (EXPERIMENTAL): Idempotency enforcement for client requests. This deduplicates requests where the client intends to send one request to Parse Server but due to network issues the server receives the request multiple times. **Caution, this is an experimental feature that may not be appropriate for production.** [#6744](https://github.com/parse-community/parse-server/issues/6744). Thanks to [Manuel Trezza](https://github.com/mtrezza).
 
 ### 4.2.0
 [Full Changelog](https://github.com/parse-community/parse-server/compare/4.1.0...4.2.0)

README.md

@@ -388,6 +388,39 @@ Parse Server allows developers to choose from several options when hosting files
 
 `GridFSBucketAdapter` is used by default and requires no setup, but if you're interested in using S3 or Google Cloud Storage, additional configuration information is available in the [Parse Server guide](http://docs.parseplatform.org/parse-server/guide/#configuring-file-adapters).
 
+### Idempodency Enforcement
+
+**Caution, this is an experimental feature that may not be appropriate for production.**
+
+This feature deduplicates identical requests that are received by Parse Server mutliple times, typically due to network issues or network adapter access restrictions on mobile operating systems.
+
+Identical requests are identified by their request header `X-Parse-Request-Id`. Therefore a client request has to include this header for deduplication to be applied. Requests that do not contain this header cannot be deduplicated and are processed normally by Parse Server. This means rolling out this feature to clients is seamless as Parse Server still processes request without this header when this feature is enbabled.
+
+> This feature needs to be enabled on the client side to send the header and on the server to process the header. Refer to the specific Parse SDK docs to see whether the feature is supported yet.
+
+Deduplication is only done for object creation and update (`POST` and `PUT` requests). Deduplication is not done for object finding and deletion (`GET` and `DELETE` requests), as these operations are already idempotent by definition.
+
+#### Configuration example
+```
+let api = new ParseServer({
+    idempotencyOptions: {
+        paths: [".*"],       // enforce for all requests
+        ttl: 120             // keep request IDs for 120s
+    }
+}
+```
+#### Parameters
+
+| Parameter | Optional | Type  | Default value | Example values | Environment variable | Description |
+|-----------|----------|--------|---------------|-----------|-----------|-------------|
+| `idempotencyOptions` | yes | `Object` | `undefined` |   | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_OPTIONS | Setting this enables idempotency enforcement for the specified paths. |
+| `idempotencyOptions.paths`| yes | `Array<String>`  | `[]` |  `.*` (all paths, includes the examples below), <br>`functions/.*` (all functions), <br>`jobs/.*` (all jobs), <br>`classes/.*` (all classes), <br>`functions/.*` (all functions), <br>`users` (user creation / update), <br>`installations` (installation creation / update) | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_PATHS | An array of path patterns that have to match the request path for request deduplication to be enabled. The mount path must not be included, for example to match the request path `/parse/functions/myFunction` specifiy the path pattern `functions/myFunction`. A trailing slash of the request path is ignored, for example the path pattern `functions/myFunction` matches both `/parse/functions/myFunction` and `/parse/functions/myFunction/`. |
+| `idempotencyOptions.ttl` | yes | `Integer` | `300` | `60` (60 seconds) | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_TTL | The duration in seconds after which a request record is discarded from the database. Duplicate requests due to network issues can be expected to arrive within milliseconds up to several seconds. This value must be greater than `0`. |
+
+#### Notes
+
+- This feature is currently only available for MongoDB and not for Postgres.
+
 ### Logging
 
 Parse Server will, by default, log:

resources/buildConfigDefinitions.js

@@ -52,6 +52,9 @@ function getENVPrefix(iface) {
   if (iface.id.name === 'LiveQueryOptions') {
     return 'PARSE_SERVER_LIVEQUERY_';
   }
+  if (iface.id.name === 'IdempotencyOptions') {
+    return 'PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_';
+  }
 }
 
 function processProperty(property, iface) {
@@ -170,6 +173,13 @@ function parseDefaultValue(elt, value, t) {
       });
       literalValue = t.objectExpression(props);
     }
+    if (type == 'IdempotencyOptions') {
+      const object = parsers.objectParser(value);
+      const props = Object.keys(object).map((key) => {
+        return t.objectProperty(key, object[value]);
+      });
+      literalValue = t.objectExpression(props);
+    }
     if (type == 'ProtectedFields') {
       const prop = t.objectProperty(
         t.stringLiteral('_User'), t.objectPattern([

spec/Idempotency.spec.js

@@ -0,0 +1,247 @@
+'use strict';
+const Config = require('../lib/Config');
+const Definitions = require('../lib/Options/Definitions');
+const request = require('../lib/request');
+const rest = require('../lib/rest');
+const auth = require('../lib/Auth');
+const uuid = require('uuid');
+
+describe_only_db('mongo')('Idempotency', () => {
+  // Parameters
+  /** Enable TTL expiration simulated by removing entry instead of waiting for MongoDB TTL monitor which
+   runs only every 60s, so it can take up to 119s until entry removal - ain't nobody got time for that */
+  const SIMULATE_TTL = true;
+  // Helpers
+  async function deleteRequestEntry(reqId) {
+    const config = Config.get(Parse.applicationId);
+    const res = await rest.find(
+      config,
+      auth.master(config),
+      '_Idempotency',
+      { reqId: reqId },
+      { limit: 1 }
+    );
+    await rest.del(
+      config,
+      auth.master(config),
+      '_Idempotency',
+      res.results[0].objectId);
+  }
+  async function setup(options) {
+    await reconfigureServer({
+      appId: Parse.applicationId,
+      masterKey: Parse.masterKey,
+      serverURL: Parse.serverURL,
+      idempotencyOptions: options,
+    });
+  }
+  // Setups
+  beforeEach(async () => {
+    if (SIMULATE_TTL) { jasmine.DEFAULT_TIMEOUT_INTERVAL = 200000; }
+    await setup({
+      paths: [
+        "functions/.*",
+        "jobs/.*",
+        "classes/.*",
+        "users",
+        "installations"
+      ],
+      ttl: 30,
+    });
+  });
+  // Tests
+  it('should enforce idempotency for cloud code function', async () => {
+    let counter = 0;
+    Parse.Cloud.define('myFunction', () => {
+      counter++;
+    });
+    const params = {
+      method: 'POST',
+      url: 'http://localhost:8378/1/functions/myFunction',
+      headers: {
+        'X-Parse-Application-Id': Parse.applicationId,
+        'X-Parse-Master-Key': Parse.masterKey,
+        'X-Parse-Request-Id': 'abc-123'
+      }
+    };
+    expect(Config.get(Parse.applicationId).idempotencyOptions.ttl).toBe(30);
+    await request(params);
+    await request(params).then(fail, e => {
+      expect(e.status).toEqual(400);
+      expect(e.data.error).toEqual("Duplicate request");
+    });
+    expect(counter).toBe(1);
+  });
+
+  it('should delete request entry after TTL', async () => {
+    let counter = 0;
+    Parse.Cloud.define('myFunction', () => {
+      counter++;
+    });
+    const params = {
+      method: 'POST',
+      url: 'http://localhost:8378/1/functions/myFunction',
+      headers: {
+        'X-Parse-Application-Id': Parse.applicationId,
+        'X-Parse-Master-Key': Parse.masterKey,
+        'X-Parse-Request-Id': 'abc-123'
+      }
+    };
+    await expectAsync(request(params)).toBeResolved();
+    if (SIMULATE_TTL) {
+      await deleteRequestEntry('abc-123');
+    } else {
+      await new Promise(resolve => setTimeout(resolve, 130000));
+    }
+    await expectAsync(request(params)).toBeResolved();
+    expect(counter).toBe(2);
+  });
+
+  it('should enforce idempotency for cloud code jobs', async () => {
+    let counter = 0;
+    Parse.Cloud.job('myJob', () => {
+      counter++;
+    });
+    const params = {
+      method: 'POST',
+      url: 'http://localhost:8378/1/jobs/myJob',
+      headers: {
+        'X-Parse-Application-Id': Parse.applicationId,
+        'X-Parse-Master-Key': Parse.masterKey,
+        'X-Parse-Request-Id': 'abc-123'
+      }
+    };
+    await expectAsync(request(params)).toBeResolved();
+    await request(params).then(fail, e => {
+      expect(e.status).toEqual(400);
+      expect(e.data.error).toEqual("Duplicate request");
+    });
+    expect(counter).toBe(1);
+  });
+
+  it('should enforce idempotency for class object creation', async () => {
+    let counter = 0;
+    Parse.Cloud.afterSave('MyClass', () => {
+      counter++;
+    });
+    const params = {
+      method: 'POST',
+      url: 'http://localhost:8378/1/classes/MyClass',
+      headers: {
+        'X-Parse-Application-Id': Parse.applicationId,
+        'X-Parse-Master-Key': Parse.masterKey,
+        'X-Parse-Request-Id': 'abc-123'
+      }
+    };
+    await expectAsync(request(params)).toBeResolved();
+    await request(params).then(fail, e => {
+      expect(e.status).toEqual(400);
+      expect(e.data.error).toEqual("Duplicate request");
+    });
+    expect(counter).toBe(1);
+  });
+
+  it('should enforce idempotency for user object creation', async () => {
+    let counter = 0;
+    Parse.Cloud.afterSave('_User', () => {
+      counter++;
+    });
+    const params = {
+      method: 'POST',
+      url: 'http://localhost:8378/1/users',
+      body: {
+        username: "user",
+        password: "pass"
+      },
+      headers: {
+        'X-Parse-Application-Id': Parse.applicationId,
+        'X-Parse-Master-Key': Parse.masterKey,
+        'X-Parse-Request-Id': 'abc-123'
+      }
+    };
+    await expectAsync(request(params)).toBeResolved();
+    await request(params).then(fail, e => {
+      expect(e.status).toEqual(400);
+      expect(e.data.error).toEqual("Duplicate request");
+    });
+    expect(counter).toBe(1);
+  });
+
+  it('should enforce idempotency for installation object creation', async () => {
+    let counter = 0;
+    Parse.Cloud.afterSave('_Installation', () => {
+      counter++;
+    });
+    const params = {
+      method: 'POST',
+      url: 'http://localhost:8378/1/installations',
+      body: {
+        installationId: "1",
+        deviceType: "ios"
+      },
+      headers: {
+        'X-Parse-Application-Id': Parse.applicationId,
+        'X-Parse-Master-Key': Parse.masterKey,
+        'X-Parse-Request-Id': 'abc-123'
+      }
+    };
+    await expectAsync(request(params)).toBeResolved();
+    await request(params).then(fail, e => {
+      expect(e.status).toEqual(400);
+      expect(e.data.error).toEqual("Duplicate request");
+    });
+    expect(counter).toBe(1);
+  });
+
+  it('should not interfere with calls of different request ID', async () => {
+    let counter = 0;
+    Parse.Cloud.afterSave('MyClass', () => {
+      counter++;
+    });
+    const promises = [...Array(100).keys()].map(() => {
+      const params = {
+        method: 'POST',
+        url: 'http://localhost:8378/1/classes/MyClass',
+        headers: {
+          'X-Parse-Application-Id': Parse.applicationId,
+          'X-Parse-Master-Key': Parse.masterKey,
+          'X-Parse-Request-Id': uuid.v4()
+        }
+      };
+      return request(params);
+    });
+    await expectAsync(Promise.all(promises)).toBeResolved();
+    expect(counter).toBe(100);
+  });
+
+  it('should re-throw any other error unchanged when writing request entry fails for any other reason', async () => {
+    spyOn(rest, 'create').and.rejectWith(new Parse.Error(0, "some other error"));
+    Parse.Cloud.define('myFunction', () => {});
+    const params = {
+      method: 'POST',
+      url: 'http://localhost:8378/1/functions/myFunction',
+      headers: {
+        'X-Parse-Application-Id': Parse.applicationId,
+        'X-Parse-Master-Key': Parse.masterKey,
+        'X-Parse-Request-Id': 'abc-123'
+      }
+    };
+    await request(params).then(fail, e => {
+      expect(e.status).toEqual(400);
+      expect(e.data.error).toEqual("some other error");
+    });
+  });
+
+  it('should use default configuration when none is set', async () => {
+    await setup({});
+    expect(Config.get(Parse.applicationId).idempotencyOptions.ttl).toBe(Definitions.IdempotencyOptions.ttl.default);
+    expect(Config.get(Parse.applicationId).idempotencyOptions.paths).toBe(Definitions.IdempotencyOptions.paths.default);
+  });
+
+  it('should throw on invalid configuration', async () => {
+    await expectAsync(setup({ paths: 1 })).toBeRejected();
+    await expectAsync(setup({ ttl: 'a' })).toBeRejected();
+    await expectAsync(setup({ ttl: 0 })).toBeRejected();
+    await expectAsync(setup({ ttl: -1 })).toBeRejected();
+  });
+});

spec/ParseQuery.Aggregate.spec.js

@@ -1440,7 +1440,7 @@ describe('Parse.Query Aggregate testing', () => {
       ['location'],
       'geoIndex',
       false,
-      '2dsphere'
+      { indexType: '2dsphere' },
     );
     // Create objects
     const GeoObject = Parse.Object.extend('GeoObject');

spec/helper.js

@@ -230,6 +230,7 @@ afterEach(function(done) {
                   '_Session',
                   '_Product',
                   '_Audience',
+                  '_Idempotency'
                 ].indexOf(className) >= 0
               );
             }

src/Adapters/Storage/Mongo/MongoStorageAdapter.js

@@ -692,26 +692,28 @@ export class MongoStorageAdapter implements StorageAdapter {
     fieldNames: string[],
     indexName: ?string,
     caseInsensitive: boolean = false,
-    indexType: any = 1
+    options?: Object = {},
   ): Promise<any> {
     schema = convertParseSchemaToMongoSchema(schema);
     const indexCreationRequest = {};
     const mongoFieldNames = fieldNames.map((fieldName) =>
       transformKey(className, fieldName, schema)
     );
     mongoFieldNames.forEach((fieldName) => {
-      indexCreationRequest[fieldName] = indexType;
+      indexCreationRequest[fieldName] = options.indexType !== undefined ? options.indexType : 1;
     });
 
     const defaultOptions: Object = { background: true, sparse: true };
     const indexNameOptions: Object = indexName ? { name: indexName } : {};
+    const ttlOptions: Object = options.ttl !== undefined ? { expireAfterSeconds: options.ttl } : {};
     const caseInsensitiveOptions: Object = caseInsensitive
       ? { collation: MongoCollection.caseInsensitiveCollation() }
       : {};
     const indexOptions: Object = {
       ...defaultOptions,
       ...caseInsensitiveOptions,
       ...indexNameOptions,
+      ...ttlOptions,
     };
 
     return this._adaptiveCollection(className)

src/Adapters/Storage/Postgres/PostgresStorageAdapter.js

@@ -1209,6 +1209,7 @@ export class PostgresStorageAdapter implements StorageAdapter {
             '_GlobalConfig',
             '_GraphQLConfig',
             '_Audience',
+            '_Idempotency',
             ...results.map((result) => result.className),
             ...joins,
           ];
@@ -2576,9 +2577,9 @@ export class PostgresStorageAdapter implements StorageAdapter {
     fieldNames: string[],
     indexName: ?string,
     caseInsensitive: boolean = false,
-    conn: ?any = null
+    options?: Object = {},
   ): Promise<any> {
-    conn = conn != null ? conn : this._client;
+    const conn = options.conn !== undefined ? options.conn : this._client;
     const defaultIndexName = `parse_default_${fieldNames.sort().join('_')}`;
     const indexNameOptions: Object =
       indexName != null ? { name: indexName } : { name: defaultIndexName };

src/Adapters/Storage/StorageAdapter.js

@@ -93,7 +93,7 @@ export interface StorageAdapter {
     fieldNames: string[],
     indexName?: string,
     caseSensitive?: boolean,
-    indexType?: any
+    options?: Object,
   ): Promise<any>;
   ensureUniqueness(
     className: string,