Case insensitive username and email indexing and query planning for Postgres

CONTRIBUTING.md

@@ -58,27 +58,37 @@ Once you have babel running in watch mode, you can start making changes to parse
 
 If your pull request introduces a change that may affect the storage or retrieval of objects, you may want to make sure it plays nice with Postgres.
 
-* Run the tests against the postgres database with `PARSE_SERVER_TEST_DB=postgres npm test`. You'll need to have postgres running on your machine and setup [appropriately](https://github.com/parse-community/parse-server/blob/master/.travis.yml#L37) or use [`Docker`](#run-a-parse-postgres-with-docker).
+* Run the tests against the postgres database with `PARSE_SERVER_TEST_DB=postgres npm test`. You'll need to have postgres running on your machine and your configuratoin file setup [appropriately](https://github.com/postgres/postgres/blob/master/src/backend/utils/misc/postgresql.conf.sample#L63) along with [PostGIS](https://postgis.net) 2.2.0 higher installed, or use [`Docker`](#run-a-parse-postgres-with-docker).
 * The Postgres adapter has a special debugger that traces all the sql commands. You can enable it with setting the environment variable `PARSE_SERVER_LOG_LEVEL=debug`
 * If your feature is intended to only work with MongoDB, you should disable PostgreSQL-specific tests with:
 
   - `describe_only_db('mongo')` // will create a `describe` that runs only on mongoDB
   - `it_only_db('mongo')` // will make a test that only runs on mongo
   - `it_exclude_dbs(['postgres'])` // will make a test that runs against all DB's but postgres
 
-#### Run a Parse Postgres with Docker
+#### Run Postgres setup for Parse with Docker
 
-To launch the compatible Postgres instance, copy and paste the following line into your shell:
+[PostGIS images (select one with v2.2 or higher) on docker dashboard](https://hub.docker.com/r/postgis/postgis) is based off of the official [postgres](https://registry.hub.docker.com/_/postgres/) image and will work out-of-the-box (as long as you create a user with the necessary extensions for each of your Parse databases; see below). To launch the compatible Postgres instance, copy and paste the following line into your shell:
 
-```sh
-docker run -d --name parse-postgres -p 5432:5432 -e POSTGRES_USER=$USER --rm mdillon/postgis:11-alpine && sleep 5 && docker exec -it parse-postgres psql -U $USER -c 'create database parse_server_postgres_adapter_test_database;' && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis;' -d parse_server_postgres_adapter_test_database && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis_topology;' -d parse_server_postgres_adapter_test_database
+```
+docker run -d --name parse-postgres -p 5432:5432 -e POSTGRES_PASSWORD=postgres --rm postgis/postgis:11-2.5-alpine && sleep 20 && docker exec -it parse-postgres psql -U postgres -c 'CREATE DATABASE parse_server_postgres_adapter_test_database;' && docker exec -it parse-postgres psql -U postgres -c 'CREATE EXTENSION postgis;' -d parse_server_postgres_adapter_test_database && docker exec -it parse-postgres psql -U postgres -c 'CREATE EXTENSION postgis_topology;' -d parse_server_postgres_adapter_test_database
 ```
 To stop the Postgres instance:
 
-```sh
+```
 docker stop parse-postgres
 ```
 
+You can also use the [postgis/postgis:11-2.5-alpine](https://hub.docker.com/r/postgis/postgis) image in a Dockerfile and copy this [script](https://github.com/parse-community/parse-server/blob/master/scripts/before_script_postgres.sh) to the image by adding the following lines: 
+
+```
+#Install additional scripts. These are run in abc order during initial start
+COPY ./scripts/setup-dbs.sh /docker-entrypoint-initdb.d/setup-dbs.sh
+RUN chmod +x /docker-entrypoint-initdb.d/setup-dbs.sh
+```
+
+Note that the script above will ONLY be executed during initialization of the container with no data in the database, see the official [Postgres image](https://hub.docker.com/_/postgres) for details. If you want to use the script to run again be sure there is no data in the /var/lib/postgresql/data of the container.
+
 ### Generate Parse Server Config Definition
 
 If you want to make changes to [Parse Server Configuration][config] add the desired configuration to [src/Options/index.js][config-index] and run `npm run definitions`. This will output [src/Options/Definitions.js][config-def] and [src/Options/docs.js][config-docs]. 

README.md

@@ -79,7 +79,7 @@ The fastest and easiest way to get started is to run MongoDB and Parse Server lo
 Before you start make sure you have installed:
 
 - [NodeJS](https://www.npmjs.com/) that includes `npm`
-- [MongoDB](https://www.mongodb.com/) or [PostgreSQL](https://www.postgresql.org/)
+- [MongoDB](https://www.mongodb.com/) or [PostgreSQL](https://www.postgresql.org/)(with [PostGIS](https://postgis.net) 2.2.0 or higher)
 - Optionally [Docker](https://www.docker.com/)
 
 ### Locally

spec/PostgresStorageAdapter.spec.js

@@ -85,6 +85,7 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => {
         expect(columns).toContain('columnA');
         expect(columns).toContain('columnB');
         expect(columns).toContain('columnC');
+        dropTable(client, className);
         done();
       })
       .catch(error => done.fail(error));
@@ -113,6 +114,7 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => {
         expect(columns.length).toEqual(2);
         expect(columns).toContain('columnA');
         expect(columns).toContain('columnB');
+        dropTable(client, className);
         done();
       })
       .catch(done);
@@ -144,6 +146,211 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => {
       undefined
     );
   });
+
+  it('should use index for caseInsensitive query', async () => {
+    const tableName = '_User';
+    const schema = {
+      fields: {
+        objectId: { type: 'String' },
+        username: { type: 'String' },
+        email: { type: 'String' },
+      },
+    };
+    const client = adapter._client;
+    await dropTable(client, tableName);
+    await adapter.createTable(tableName, schema);
+    await client.none(
+      'INSERT INTO $1:name ($2:name, $3:name) VALUES ($4, $5)',
+      [tableName, 'objectId', 'username', 'Bugs', 'Bunny']
+    );
+    //Postgres won't take advantage of the index until it has a lot of records because sequential is faster for small db's
+    await client.none(
+      'INSERT INTO $1:name ($2:name, $3:name) SELECT MD5(random()::text), MD5(random()::text) FROM generate_series(1,5000)',
+      [tableName, 'objectId', 'username']
+    );
+    const caseInsensitiveData = 'bugs';
+    const originalQuery =
+      'SELECT * FROM $1:name WHERE lower($2:name)=lower($3)';
+    const analyzedExplainQuery = adapter.createExplainableQuery(
+      originalQuery,
+      true
+    );
+    await client
+      .one(analyzedExplainQuery, [tableName, 'objectId', caseInsensitiveData])
+      .then(explained => {
+        const preIndexPlan = explained;
+
+        //Make sure search returned with only 1 result
+        expect(preIndexPlan['QUERY PLAN'][0].Plan['Actual Rows']).toBe(1);
+        expect(preIndexPlan['QUERY PLAN'][0].Plan['Node Type']).toBe(
+          'Seq Scan'
+        );
+        const indexName = 'test_case_insensitive_column';
+
+        adapter
+          .ensureIndex(tableName, schema, ['objectId'], indexName, true)
+          .then(() => {
+            client
+              .one(analyzedExplainQuery, [
+                tableName,
+                'objectId',
+                caseInsensitiveData,
+              ])
+              .then(explained => {
+                const postIndexPlan = explained;
+
+                //Make sure search returned with only 1 result
+                expect(postIndexPlan['QUERY PLAN'][0].Plan['Actual Rows']).toBe(
+                  1
+                );
+                //Should not be a sequential scan
+                expect(
+                  postIndexPlan['QUERY PLAN'][0].Plan['Node Type']
+                ).not.toContain('Seq Scan');
+
+                //Should be using the index created for this
+                expect(
+                  postIndexPlan['QUERY PLAN'][0].Plan.Plans[0]['Index Name']
+                ).toBe(indexName);
+
+                //Sequential should take more time to execute than indexed
+                expect(
+                  preIndexPlan['QUERY PLAN'][0]['Execution Time']
+                ).toBeGreaterThan(
+                  postIndexPlan['QUERY PLAN'][0]['Execution Time']
+                );
+
+                //Test explaining without analyzing
+                const basicExplainQuery = adapter.createExplainableQuery(
+                  originalQuery
+                );
+                client
+                  .one(basicExplainQuery, [
+                    tableName,
+                    'objectId',
+                    caseInsensitiveData,
+                  ])
+                  .then(explained => {
+                    //Check that basic query plans isn't a sequential scan
+                    expect(
+                      explained['QUERY PLAN'][0].Plan['Node Type']
+                    ).not.toContain('Seq Scan');
+
+                    //Basic query plans shouldn't have an execution time
+                    expect(
+                      explained['QUERY PLAN'][0]['Execution Time']
+                    ).toBeUndefined();
+
+                    //Delete generated data in postgres
+                    return dropTable(client, tableName);
+                  });
+              });
+          });
+      })
+      .catch(error => {
+        // Query on non existing table, don't crash
+        if (error.code !== '42P01') {
+          throw error;
+        }
+        return [];
+      });
+  });
+
+  it('should use index for caseInsensitive query using Parse find', async () => {
+    const tableName = '_User';
+    const user = new Parse.User();
+    user.set('username', 'Bugs');
+    user.set('password', 'Bunny');
+    await user.signUp();
+    const database = Config.get(Parse.applicationId).database;
+
+    //Postgres won't take advantage of the index until it has a lot of records because sequential is faster for small db's
+    const client = adapter._client;
+    await client.none(
+      'INSERT INTO $1:name ($2:name, $3:name) SELECT MD5(random()::text), MD5(random()::text) FROM generate_series(1,5000)',
+      [tableName, 'objectId', 'username']
+    );
+    const caseInsensitiveData = 'bugs';
+    const fieldToSearch = 'username';
+    //Check using find method for Parse
+    const preIndexPlan = await database.find(
+      tableName,
+      { username: caseInsensitiveData },
+      { caseInsensitive: true, explain: true }
+    );
+
+    //Check that basic query plans isn't a sequential scan, be careful as find uses "any" to query
+    expect(
+      preIndexPlan[0]['QUERY PLAN'][0].Plan['Node Type']).toBe('Seq Scan'
+    );
+
+    //Basic query plans shouldn't have an execution time
+    expect(
+      preIndexPlan[0]['QUERY PLAN'][0]['Execution Time']
+    ).toBeUndefined();
+
+    const indexName = 'test_case_insensitive_column';
+    const schema = await new Parse.Schema('_User').get();
+    await adapter
+      .ensureIndex(tableName, schema, [fieldToSearch], indexName, true);
+
+    //Check using find method for Parse
+    const postIndexPlan = await database.find(
+      tableName,
+      { username: caseInsensitiveData },
+      { caseInsensitive: true, explain: true }
+    );
+
+    //Check that basic query plans isn't a sequential scan
+    expect(
+      postIndexPlan[0]['QUERY PLAN'][0].Plan['Node Type']
+    ).not.toContain('Seq Scan');
+
+    //Basic query plans shouldn't have an execution time
+    expect(
+      postIndexPlan[0]['QUERY PLAN'][0]['Execution Time']
+    ).toBeUndefined();
+    //Delete generated data in postgres by dropping table
+    return dropTable(client, tableName);
+  });
+
+  it('should use index for caseInsensitive query using default indexname', async () => {
+    const tableName = '_User';
+    const user = new Parse.User();
+    user.set('username', 'Bugs');
+    user.set('password', 'Bunny');
+    await user.signUp();
+    const database = Config.get(Parse.applicationId).database;
+    const fieldToSearch = 'username';
+    //Create index before data is inserted
+    const schema = await new Parse.Schema('_User').get();
+    await adapter
+      .ensureIndex(tableName, schema, [fieldToSearch], null, true);
+
+    //Postgres won't take advantage of the index until it has a lot of records because sequential is faster for small db's
+    const client = adapter._client;
+    await client.none(
+      'INSERT INTO $1:name ($2:name, $3:name) SELECT MD5(random()::text), MD5(random()::text) FROM generate_series(1,5000)',
+      [tableName, 'objectId', 'username']
+    );
+
+    const caseInsensitiveData = 'buGs';
+    //Check using find method for Parse
+    const indexPlan = await database.find(
+      tableName,
+      { username: caseInsensitiveData },
+      { caseInsensitive: true, explain: true }
+    );
+
+    expect(indexPlan[0]['QUERY PLAN'][0].Plan['Node Type']).not.toContain(
+      'Seq Scan'
+    );
+    expect(
+      indexPlan[0]['QUERY PLAN'][0].Plan['Index Name']
+    ).toContain('parse_default');
+    //Delete generated data in postgres by dropping table
+    return dropTable(client, tableName);
+  });
 });
 
 describe_only_db('postgres')('PostgresStorageAdapter shutdown', () => {