Case insensitive username and email indexing and query planning for Postgres

CONTRIBUTING.md

@@ -61,7 +61,7 @@ If your pull request introduces a change that may affect the storage or retrieva
 * Run the tests against the postgres database with `PARSE_SERVER_TEST_DB=postgres PARSE_SERVER_TEST_DATABASE_URI=postgres://postgres:password@localhost:5432/parse_server_postgres_adapter_test_database npm run testonly`. You'll need to have postgres running on your machine and setup [appropriately](https://github.com/parse-community/parse-server/blob/master/.travis.yml#L43) or use [`Docker`](#run-a-parse-postgres-with-docker).
 * The Postgres adapter has a special debugger that traces all the sql commands. You can enable it with setting the environment variable `PARSE_SERVER_LOG_LEVEL=debug`
 * If your feature is intended to only work with MongoDB, you should disable PostgreSQL-specific tests with:
-   
+
   - `describe_only_db('mongo')` // will create a `describe` that runs only on mongoDB
   - `it_only_db('mongo')` // will make a test that only runs on mongo
   - `it_exclude_dbs(['postgres'])` // will make a test that runs against all DB's but postgres
@@ -71,22 +71,32 @@ If your pull request introduces a change that may affect the storage or retrieva
   - `it_only_db('postgres')` // will make a test that only runs on postgres
   - `it_exclude_dbs(['mongo'])` // will make a test that runs against all DB's but mongo
 
-#### Run a Parse Postgres with Docker
+#### Run Postgres setup for Parse with Docker
 
-To launch the compatible Postgres instance, copy and paste the following line into your shell:
+[PostGIS images (select one with v2.2 or higher) on docker dashboard](https://hub.docker.com/r/postgis/postgis) is based off of the official [postgres](https://registry.hub.docker.com/_/postgres/) image and will work out-of-the-box (as long as you create a user with the necessary extensions for each of your Parse databases; see below). To launch the compatible Postgres instance, copy and paste the following line into your shell:
 
-```sh
-docker run -d --name parse-postgres -p 5432:5432 -e POSTGRES_USER=$USER --rm mdillon/postgis:11-alpine && sleep 5 && docker exec -it parse-postgres psql -U $USER -c 'create database parse_server_postgres_adapter_test_database;' && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis;' -d parse_server_postgres_adapter_test_database && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis_topology;' -d parse_server_postgres_adapter_test_database
+```
+docker run -d --name parse-postgres -p 5432:5432 -e POSTGRES_PASSWORD=password --rm postgis/postgis:11-3.0-alpine && sleep 20 && docker exec -it parse-postgres psql -U postgres -c 'CREATE DATABASE parse_server_postgres_adapter_test_database;' && docker exec -it parse-postgres psql -U postgres -c 'CREATE EXTENSION postgis;' -d parse_server_postgres_adapter_test_database && docker exec -it parse-postgres psql -U postgres -c 'CREATE EXTENSION postgis_topology;' -d parse_server_postgres_adapter_test_database
 ```
 To stop the Postgres instance:
 
-```sh
+```
 docker stop parse-postgres
 ```
 
+You can also use the [postgis/postgis:11-2.5-alpine](https://hub.docker.com/r/postgis/postgis) image in a Dockerfile and copy this [script](https://github.com/parse-community/parse-server/blob/master/scripts/before_script_postgres.sh) to the image by adding the following lines:
+
+```
+#Install additional scripts. These are run in abc order during initial start
+COPY ./scripts/setup-dbs.sh /docker-entrypoint-initdb.d/setup-dbs.sh
+RUN chmod +x /docker-entrypoint-initdb.d/setup-dbs.sh
+```
+
+Note that the script above will ONLY be executed during initialization of the container with no data in the database, see the official [Postgres image](https://hub.docker.com/_/postgres) for details. If you want to use the script to run again be sure there is no data in the /var/lib/postgresql/data of the container.
+
 ### Generate Parse Server Config Definition
 
-If you want to make changes to [Parse Server Configuration][config] add the desired configuration to [src/Options/index.js][config-index] and run `npm run definitions`. This will output [src/Options/Definitions.js][config-def] and [src/Options/docs.js][config-docs]. 
+If you want to make changes to [Parse Server Configuration][config] add the desired configuration to [src/Options/index.js][config-index] and run `npm run definitions`. This will output [src/Options/Definitions.js][config-def] and [src/Options/docs.js][config-docs].
 
 To view docs run `npm run docs` and check the `/out` directory.
 

README.md

@@ -79,7 +79,7 @@ The fastest and easiest way to get started is to run MongoDB and Parse Server lo
 Before you start make sure you have installed:
 
 - [NodeJS](https://www.npmjs.com/) that includes `npm`
-- [MongoDB](https://www.mongodb.com/) or [PostgreSQL](https://www.postgresql.org/)
+- [MongoDB](https://www.mongodb.com/) or [PostgreSQL](https://www.postgresql.org/)(with [PostGIS](https://postgis.net) 2.2.0 or higher)
 - Optionally [Docker](https://www.docker.com/)
 
 ### Locally

spec/PostgresStorageAdapter.spec.js

@@ -86,6 +86,7 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => {
         expect(columns).toContain('columnA');
         expect(columns).toContain('columnB');
         expect(columns).toContain('columnC');
+
         done();
       })
       .catch(error => done.fail(error));
@@ -145,6 +146,206 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => {
       undefined
     );
   });
+
+  it('should use index for caseInsensitive query using Postgres', async () => {
+    const tableName = '_User';
+    const schema = {
+      fields: {
+        objectId: { type: 'String' },
+        username: { type: 'String' },
+        email: { type: 'String' },
+      },
+    };
+    const client = adapter._client;
+    await adapter.createTable(tableName, schema);
+    await client.none(
+      'INSERT INTO $1:name ($2:name, $3:name) VALUES ($4, $5)',
+      [tableName, 'objectId', 'username', 'Bugs', 'Bunny']
+    );
+    //Postgres won't take advantage of the index until it has a lot of records because sequential is faster for small db's
+    await client.none(
+      'INSERT INTO $1:name ($2:name, $3:name) SELECT MD5(random()::text), MD5(random()::text) FROM generate_series(1,5000)',
+      [tableName, 'objectId', 'username']
+    );
+    const caseInsensitiveData = 'bugs';
+    const originalQuery =
+      'SELECT * FROM $1:name WHERE lower($2:name)=lower($3)';
+    const analyzedExplainQuery = adapter.createExplainableQuery(
+      originalQuery,
+      true
+    );
+    await client
+      .one(analyzedExplainQuery, [tableName, 'objectId', caseInsensitiveData])
+      .then(explained => {
+        const preIndexPlan = explained;
+
+        preIndexPlan['QUERY PLAN'].forEach(element => {
+          //Make sure search returned with only 1 result
+          expect(element.Plan['Actual Rows']).toBe(1);
+          expect(element.Plan['Node Type']).toBe('Seq Scan');
+        });
+        const indexName = 'test_case_insensitive_column';
+
+        adapter
+          .ensureIndex(tableName, schema, ['objectId'], indexName, true)
+          .then(() => {
+            client
+              .one(analyzedExplainQuery, [
+                tableName,
+                'objectId',
+                caseInsensitiveData,
+              ])
+              .then(explained => {
+                const postIndexPlan = explained;
+
+                postIndexPlan['QUERY PLAN'].forEach(element => {
+                  //Make sure search returned with only 1 result
+                  expect(element.Plan['Actual Rows']).toBe(1);
+                  //Should not be a sequential scan
+                  expect(element.Plan['Node Type']).not.toContain('Seq Scan');
+
+                  //Should be using the index created for this
+                  element.Plan.Plans.forEach(innerElement => {
+                    expect(innerElement['Index Name']).toBe(indexName);
+                  });
+                });
+
+                //These are the same query so should be the same size
+                for (let i = 0; i < preIndexPlan['QUERY PLAN'].length; i++) {
+                  //Sequential should take more time to execute than indexed
+                  expect(
+                    preIndexPlan['QUERY PLAN'][i]['Execution Time']
+                  ).toBeGreaterThan(
+                    postIndexPlan['QUERY PLAN'][i]['Execution Time']
+                  );
+                }
+
+                //Test explaining without analyzing
+                const basicExplainQuery = adapter.createExplainableQuery(
+                  originalQuery
+                );
+                client
+                  .one(basicExplainQuery, [
+                    tableName,
+                    'objectId',
+                    caseInsensitiveData,
+                  ])
+                  .then(explained => {
+                    explained['QUERY PLAN'].forEach(element => {
+                      //Check that basic query plans isn't a sequential scan
+                      expect(element.Plan['Node Type']).not.toContain(
+                        'Seq Scan'
+                      );
+
+                      //Basic query plans shouldn't have an execution time
+                      expect(element['Execution Time']).toBeUndefined();
+                    });
+                  });
+              });
+          });
+      })
+      .catch(error => {
+        // Query on non existing table, don't crash
+        if (error.code !== '42P01') {
+          throw error;
+        }
+        return [];
+      });
+  });
+
+  it('should use index for caseInsensitive query', async () => {
+    const tableName = '_User';
+    const user = new Parse.User();
+    user.set('username', 'Bugs');
+    user.set('password', 'Bunny');
+    await user.signUp();
+    const database = Config.get(Parse.applicationId).database;
+
+    //Postgres won't take advantage of the index until it has a lot of records because sequential is faster for small db's
+    const client = adapter._client;
+    await client.none(
+      'INSERT INTO $1:name ($2:name, $3:name) SELECT MD5(random()::text), MD5(random()::text) FROM generate_series(1,5000)',
+      [tableName, 'objectId', 'username']
+    );
+    const caseInsensitiveData = 'bugs';
+    const fieldToSearch = 'username';
+    //Check using find method for Parse
+    const preIndexPlan = await database.find(
+      tableName,
+      { username: caseInsensitiveData },
+      { caseInsensitive: true, explain: true }
+    );
+
+    preIndexPlan.forEach(element => {
+      element['QUERY PLAN'].forEach(innerElement => {
+        //Check that basic query plans isn't a sequential scan, be careful as find uses "any" to query
+        expect(innerElement.Plan['Node Type']).toBe('Seq Scan');
+        //Basic query plans shouldn't have an execution time
+        expect(innerElement['Execution Time']).toBeUndefined();
+      });
+    });
+
+    const indexName = 'test_case_insensitive_column';
+    const schema = await new Parse.Schema('_User').get();
+    await adapter.ensureIndex(
+      tableName,
+      schema,
+      [fieldToSearch],
+      indexName,
+      true
+    );
+
+    //Check using find method for Parse
+    const postIndexPlan = await database.find(
+      tableName,
+      { username: caseInsensitiveData },
+      { caseInsensitive: true, explain: true }
+    );
+
+    postIndexPlan.forEach(element => {
+      element['QUERY PLAN'].forEach(innerElement => {
+        //Check that basic query plans isn't a sequential scan
+        expect(innerElement.Plan['Node Type']).not.toContain('Seq Scan');
+
+        //Basic query plans shouldn't have an execution time
+        expect(innerElement['Execution Time']).toBeUndefined();
+      });
+    });
+  });
+
+  it('should use index for caseInsensitive query using default indexname', async () => {
+    const tableName = '_User';
+    const user = new Parse.User();
+    user.set('username', 'Bugs');
+    user.set('password', 'Bunny');
+    await user.signUp();
+    const database = Config.get(Parse.applicationId).database;
+    const fieldToSearch = 'username';
+    //Create index before data is inserted
+    const schema = await new Parse.Schema('_User').get();
+    await adapter.ensureIndex(tableName, schema, [fieldToSearch], null, true);
+
+    //Postgres won't take advantage of the index until it has a lot of records because sequential is faster for small db's
+    const client = adapter._client;
+    await client.none(
+      'INSERT INTO $1:name ($2:name, $3:name) SELECT MD5(random()::text), MD5(random()::text) FROM generate_series(1,5000)',
+      [tableName, 'objectId', 'username']
+    );
+
+    const caseInsensitiveData = 'buGs';
+    //Check using find method for Parse
+    const indexPlan = await database.find(
+      tableName,
+      { username: caseInsensitiveData },
+      { caseInsensitive: true, explain: true }
+    );
+    indexPlan.forEach(element => {
+      element['QUERY PLAN'].forEach(innerElement => {
+        expect(innerElement.Plan['Node Type']).not.toContain('Seq Scan');
+        expect(innerElement.Plan['Index Name']).toContain('parse_default');
+      });
+    });
+  });
 });
 
 describe_only_db('postgres')('PostgresStorageAdapter shutdown', () => {