Fix completion refresher query results leak and active query error (dbcli#161)

* Adding a lock acquisition during executing a single batch query.

* Fixing casing in order to retrieve proper error message.

* Adding support for owner uri in json rpc response dequeueing.

* Adding support for a separate connection for the completion refrehser with addt'l logging.

* Marking string as unicode for py2/3 compat.

* Converging all sqltoolsservice test to a single file since most of the code was redundant and hard to maintain.

* Adding missing json rpc client tests and reformatted for readability.

* Adding clean up for each request to remove it's owner uri from response queue when it's completed.

* Adding request finished for owner uri of request when a exception occurs.

* Setting response_str to None to avoid assignment exception.

* Fixing build.py format.

* Adding owner URI to connection complete event during failure.

* Changing get response signature in jsonrpcclient to accept owner uri in addition to id.

* Removing copyright headers from files I touched.

* Refactored test json rpc contracts tests.

* Adding unit test for ensuring query execute requests retrieve correct events based on it's owner uri.

* Fixing typo in refresher name.

* Renaming method for creating a new mssqlcliclient for completion refresher.

* Renamed baseline files for simplicity.

* Removed copyright header.

* Refactoring and adding clone support to mssqlcliclient.

* Removing unnecessary call to get a new mssql cli client.

Author: MrMeemus

build.py

@@ -95,9 +95,17 @@ def unit_test():
     Run all unit tests.
     """
     utility.exec_command(
-        'pytest --cov mssqlcli tests/test_mssqlcliclient.py tests/test_main.py tests/test_fuzzy_completion.py '
-        'tests/test_rowlimit.py tests/test_sqlcompletion.py tests/test_prioritization.py mssqlcli/jsonrpc/contracts/tests '
-        'tests/test_telemetry.py tests/test_special.py',
+        'pytest --cov mssqlcli '
+        'tests/test_mssqlcliclient.py '
+        'tests/test_main.py '
+        'tests/test_fuzzy_completion.py '
+        'tests/test_rowlimit.py '
+        'tests/test_sqlcompletion.py '
+        'tests/test_prioritization.py '
+        'mssqlcli/jsonrpc/tests '
+        'mssqlcli/jsonrpc/contracts/tests '
+        'tests/test_telemetry.py '
+        'tests/test_special.py',
         utility.ROOT_DIR,
         continue_on_error=False)
 

mssqlcli/completion_refresher.py

@@ -51,7 +51,11 @@ def _bg_refresh(self, mssqlcliclient, callbacks, history=None,
         completer = MssqlCompleter(smart_completion=True, settings=settings)
 
         executor = mssqlcliclient
-        executor.connect()
+        if not executor.connect():
+            # If we were unable to connect, do not break the experience for the user.
+            # Return nothing, smart completion can maintain the keywords and functions completions.
+            logger.error(u'Completion Refresher failed to connect to the target server.')
+            return
         # If callbacks is a single function then push it into a list.
         if callable(callbacks):
             callbacks = [callbacks]

mssqlcli/jsonrpc/contracts/connectionservice.py

@@ -1,7 +1,3 @@
-# --------------------------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for license information.
-# --------------------------------------------------------------------------------------------
 from mssqlcli.jsonrpc.contracts import Request
 
 import sys
@@ -16,8 +12,9 @@ class ConnectionRequest(Request):
     """
     METHOD_NAME = u'connection/connect'
 
-    def __init__(self, id, json_rpc_client, parameters):
+    def __init__(self, id, owner_uri, json_rpc_client, parameters):
         self.id = id
+        self.owner_uri = owner_uri
         self.finished = False
         self.json_rpc_client = json_rpc_client
         self.params = ConnectionParams(parameters)
@@ -31,26 +28,27 @@ def get_response(self):
             Get latest response, event or exception if it occured.
         """
         try:
-            response = self.json_rpc_client.get_response(self.id)
+            response = self.json_rpc_client.get_response(self.id, self.owner_uri)
             decoded_response = None
-
             if response:
                 logger.debug(response)
                 decoded_response = self.decode_response(response)
 
             if isinstance(decoded_response, ConnectionCompleteEvent):
                 self.finished = True
                 self.json_rpc_client.request_finished(self.id)
+                self.json_rpc_client.request_finished(self.owner_uri)
 
             return decoded_response
 
         except Exception as error:
             logger.debug(str(error))
             self.finished = True
             self.json_rpc_client.request_finished(self.id)
+            self.json_rpc_client.request_finished(self.owner_uri)
             return ConnectionCompleteEvent({
                 u'params': {
-                    u'ownerUri': None,
+                    u'ownerUri': self.owner_uri,
                     u'connectionId': None,
                     u'messages': str(error),
                     u'errorMessage': u'Connection request encountered an exception',

mssqlcli/jsonrpc/contracts/queryexecutestringservice.py

@@ -1,10 +1,8 @@
-# --------------------------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for license information.
-# --------------------------------------------------------------------------------------------
-
+import logging
 from mssqlcli.jsonrpc.contracts import Request
 
+logger = logging.getLogger(u'mssqlcli.queryexecutestringservice')
+
 
 class QueryExecuteStringRequest(Request):
     """
@@ -13,8 +11,9 @@ class QueryExecuteStringRequest(Request):
 
     METHOD_NAME = u'query/executeString'
 
-    def __init__(self, id, json_rpc_client, parameters):
+    def __init__(self, id, owner_uri, json_rpc_client, parameters):
         self.id = id
+        self.owner_uri = owner_uri
         self.finished = False
         self.json_rpc_client = json_rpc_client
         self.params = QueryExecuteStringParams(parameters)
@@ -24,22 +23,24 @@ def get_response(self):
             Get latest response, event or exception if occured.
         """
         try:
-            response = self.json_rpc_client.get_response(self.id)
-            decoded_response = None
+            response = self.json_rpc_client.get_response(self.id, self.owner_uri)
 
+            decoded_response = None
             if response:
                 decoded_response = self.decode_response(response)
 
             if isinstance(decoded_response, QueryCompleteEvent) or \
                isinstance(decoded_response, QueryExecuteErrorResponseEvent):
                 self.finished = True
                 self.json_rpc_client.request_finished(self.id)
+                self.json_rpc_client.request_finished(self.owner_uri)
 
             return decoded_response
 
         except Exception as error:
             self.finished = True
             self.json_rpc_client.request_finished(self.id)
+            self.json_rpc_client.request_finished(self.owner_uri)
             return QueryCompleteEvent(
                 {u'params': None}, exception_message=str(error)
             )
@@ -137,8 +138,9 @@ class QuerySubsetRequest(Request):
 
     METHOD_NAME = u'query/subset'
 
-    def __init__(self, id, json_rpc_client, parameters):
+    def __init__(self, id, owner_uri, json_rpc_client, parameters):
         self.id = id
+        self.owner_uri = owner_uri
         self.finished = False
         self.json_rpc_client = json_rpc_client
         self.params = QuerySubsetParams(parameters)
@@ -151,21 +153,22 @@ def completed(self):
 
     def get_response(self):
         try:
-            response = self.json_rpc_client.get_response(self.id)
+            response = self.json_rpc_client.get_response(self.id, self.owner_uri)
             decoded_response = None
-
             if response:
                 decoded_response = self.decode_response(response)
 
             if isinstance(decoded_response, ResultSubset):
                 self.finished = True
                 self.json_rpc_client.request_finished(self.id)
+                self.json_rpc_client.request_finished(self.owner_uri)
 
             return decoded_response
 
         except Exception as error:
             self.finished = True
             self.json_rpc_client.request_finished(self.id)
+            self.json_rpc_client.request_finished(self.owner_uri)
             return ResultSubset(None, error_message=str(error))
 
     def execute(self):
@@ -177,7 +180,7 @@ def decode_response(self, response):
             return ResultSubset(response)
         elif u'error' in response:
             return ResultSubset(
-                None, error_message=response[u'error']['Message'])
+                None, error_message=response[u'error']['message'])
 
 
 class QuerySubsetParams(object):

mssqlcli/jsonrpc/contracts/tests/baselines/malformed_query_adventureworks2014.txt renamed to mssqlcli/jsonrpc/contracts/tests/baselines/test_malformed_query.txt

@@ -0,0 +1,25 @@
+Content-Length: 40
+
+{"jsonrpc":"2.0","id":"1","result":true}Content-Length: 40
+
+{"jsonrpc":"2.0","id":"2","result":true}Content-Length: 650
+
+{"jsonrpc":"2.0","method":"connection/complete","params":{"ownerUri":"mismatchquerycompleteresponse_2","connectionId":"7cd08d4a-6b39-429b-9f28-bf1b42fb75c5","messages":null,"errorMessage":null,"errorNumber":0,"serverInfo":{"serverMajorVersion":12,"serverMinorVersion":0,"serverReleaseVersion":2269,"engineEditionId":3,"serverVersion":"12.0.2269.0","serverLevel":"RTM","serverEdition":"Enterprise Edition (64-bit)","isCloud":false,"azureVersion":0,"osVersion":"Windows NT 6.3 <X64> (Build 9600: ) (Hypervisor)\n","machineName":"BRO-HB"},"connectionSummary":{"serverName":"bro-hb","databaseName":"AdventureWorks2014","userName":"sa"},"type":"Default"}}Content-Length: 650
+
+{"jsonrpc":"2.0","method":"connection/complete","params":{"ownerUri":"mismatchquerycompleteresponse_1","connectionId":"eb327978-b068-4408-aeef-836f30be870a","messages":null,"errorMessage":null,"errorNumber":0,"serverInfo":{"serverMajorVersion":12,"serverMinorVersion":0,"serverReleaseVersion":2269,"engineEditionId":3,"serverVersion":"12.0.2269.0","serverLevel":"RTM","serverEdition":"Enterprise Edition (64-bit)","isCloud":false,"azureVersion":0,"osVersion":"Windows NT 6.3 <X64> (Build 9600: ) (Hypervisor)\n","machineName":"BRO-HB"},"connectionSummary":{"serverName":"bro-hb","databaseName":"AdventureWorks2014","userName":"sa"},"type":"Default"}}Content-Length: 115
+
+{"jsonrpc":"2.0","method":"textDocument/intelliSenseReady","params":{"ownerUri":"mismatchquerycompleteresponse_2"}}Content-Length: 115
+
+{"jsonrpc":"2.0","method":"textDocument/intelliSenseReady","params":{"ownerUri":"mismatchquerycompleteresponse_1"}}Content-Length: 38
+
+{"jsonrpc":"2.0","id":"3","result":{}}Content-Length: 357
+
+{"jsonrpc":"2.0","method":"query/batchStart","params":{"batchSummary":{"executionElapsed":null,"executionEnd":null,"executionStart":"2018-02-08T13:20:15.1872770-08:00","hasError":false,"id":0,"selection":{"endColumn":39,"endLine":1,"startColumn":0,"startLine":0},"resultSetSummaries":null,"specialAction":null},"ownerUri":"mismatchquerycompleteresponse_1"}}Content-Length: 212
+
+{"jsonrpc":"2.0","method":"query/message","params":{"ownerUri":"mismatchquerycompleteresponse_1","message":{"batchId":0,"isError":false,"time":"2018-02-08T13:20:15.3145728-08:00","message":"(16 rows affected)"}}}Content-Length: 3196
+
+{"jsonrpc":"2.0","method":"query/resultSetComplete","params":{"resultSetSummary":{"id":0,"batchId":0,"rowCount":16,"columnInfo":[{"isBytes":false,"isChars":false,"isSqlVariant":false,"isUdt":false,"isXml":false,"isJson":false,"sqlDbType":16,"isSqlXmlType":false,"isUnknownType":false,"isUpdatable":false,"allowDBNull":false,"baseCatalogName":null,"baseColumnName":null,"baseSchemaName":null,"baseServerName":null,"baseTableName":null,"columnName":"DepartmentID","columnOrdinal":0,"columnSize":2,"isAliased":null,"isAutoIncrement":true,"isExpression":null,"isHidden":null,"isIdentity":true,"isKey":null,"isLong":false,"isReadOnly":true,"isUnique":false,"numericPrecision":5,"numericScale":255,"udtAssemblyQualifiedName":null,"dataType":"System.Int16, System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","dataTypeName":"smallint"},{"isBytes":false,"isChars":true,"isSqlVariant":false,"isUdt":false,"isXml":false,"isJson":false,"sqlDbType":12,"isSqlXmlType":false,"isUnknownType":false,"isUpdatable":true,"allowDBNull":false,"baseCatalogName":null,"baseColumnName":null,"baseSchemaName":null,"baseServerName":null,"baseTableName":null,"columnName":"Name","columnOrdinal":1,"columnSize":50,"isAliased":null,"isAutoIncrement":false,"isExpression":null,"isHidden":null,"isIdentity":false,"isKey":null,"isLong":false,"isReadOnly":false,"isUnique":false,"numericPrecision":255,"numericScale":255,"udtAssemblyQualifiedName":null,"dataType":"System.String, System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","dataTypeName":"nvarchar"},{"isBytes":false,"isChars":true,"isSqlVariant":false,"isUdt":false,"isXml":false,"isJson":false,"sqlDbType":12,"isSqlXmlType":false,"isUnknownType":false,"isUpdatable":true,"allowDBNull":false,"baseCatalogName":null,"baseColumnName":null,"baseSchemaName":null,"baseServerName":null,"baseTableName":null,"columnName":"GroupName","columnOrdinal":2,"columnSize":50,"isAliased":null,"isAutoIncrement":false,"isExpression":null,"isHidden":null,"isIdentity":false,"isKey":null,"isLong":false,"isReadOnly":false,"isUnique":false,"numericPrecision":255,"numericScale":255,"udtAssemblyQualifiedName":null,"dataType":"System.String, System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","dataTypeName":"nvarchar"},{"isBytes":false,"isChars":false,"isSqlVariant":false,"isUdt":false,"isXml":false,"isJson":false,"sqlDbType":4,"isSqlXmlType":false,"isUnknownType":false,"isUpdatable":true,"allowDBNull":false,"baseCatalogName":null,"baseColumnName":null,"baseSchemaName":null,"baseServerName":null,"baseTableName":null,"columnName":"ModifiedDate","columnOrdinal":3,"columnSize":8,"isAliased":null,"isAutoIncrement":false,"isExpression":null,"isHidden":null,"isIdentity":false,"isKey":null,"isLong":false,"isReadOnly":false,"isUnique":false,"numericPrecision":23,"numericScale":3,"udtAssemblyQualifiedName":null,"dataType":"System.DateTime, System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","dataTypeName":"datetime"}],"specialAction":{"none":true,"expectYukonXMLShowPlan":false}},"ownerUri":"mismatchquerycompleteresponse_1"}}Content-Length: 3511
+
+{"jsonrpc":"2.0","method":"query/batchComplete","params":{"batchSummary":{"executionElapsed":"00:00:00.0805260","executionEnd":"2018-02-08T13:20:15.3449966-08:00","executionStart":"2018-02-08T13:20:15.2644706-08:00","hasError":false,"id":0,"selection":{"endColumn":39,"endLine":1,"startColumn":0,"startLine":0},"resultSetSummaries":[{"id":0,"batchId":0,"rowCount":16,"columnInfo":[{"isBytes":false,"isChars":false,"isSqlVariant":false,"isUdt":false,"isXml":false,"isJson":false,"sqlDbType":16,"isSqlXmlType":false,"isUnknownType":false,"isUpdatable":false,"allowDBNull":false,"baseCatalogName":null,"baseColumnName":null,"baseSchemaName":null,"baseServerName":null,"baseTableName":null,"columnName":"DepartmentID","columnOrdinal":0,"columnSize":2,"isAliased":null,"isAutoIncrement":true,"isExpression":null,"isHidden":null,"isIdentity":true,"isKey":null,"isLong":false,"isReadOnly":true,"isUnique":false,"numericPrecision":5,"numericScale":255,"udtAssemblyQualifiedName":null,"dataType":"System.Int16, System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","dataTypeName":"smallint"},{"isBytes":false,"isChars":true,"isSqlVariant":false,"isUdt":false,"isXml":false,"isJson":false,"sqlDbType":12,"isSqlXmlType":false,"isUnknownType":false,"isUpdatable":true,"allowDBNull":false,"baseCatalogName":null,"baseColumnName":null,"baseSchemaName":null,"baseServerName":null,"baseTableName":null,"columnName":"Name","columnOrdinal":1,"columnSize":50,"isAliased":null,"isAutoIncrement":false,"isExpression":null,"isHidden":null,"isIdentity":false,"isKey":null,"isLong":false,"isReadOnly":false,"isUnique":false,"numericPrecision":255,"numericScale":255,"udtAssemblyQualifiedName":null,"dataType":"System.String, System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","dataTypeName":"nvarchar"},{"isBytes":false,"isChars":true,"isSqlVariant":false,"isUdt":false,"isXml":false,"isJson":false,"sqlDbType":12,"isSqlXmlType":false,"isUnknownType":false,"isUpdatable":true,"allowDBNull":false,"baseCatalogName":null,"baseColumnName":null,"baseSchemaName":null,"baseServerName":null,"baseTableName":null,"columnName":"GroupName","columnOrdinal":2,"columnSize":50,"isAliased":null,"isAutoIncrement":false,"isExpression":null,"isHidden":null,"isIdentity":false,"isKey":null,"isLong":false,"isReadOnly":false,"isUnique":false,"numericPrecision":255,"numericScale":255,"udtAssemblyQualifiedName":null,"dataType":"System.String, System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","dataTypeName":"nvarchar"},{"isBytes":false,"isChars":false,"isSqlVariant":false,"isUdt":false,"isXml":false,"isJson":false,"sqlDbType":4,"isSqlXmlType":false,"isUnknownType":false,"isUpdatable":true,"allowDBNull":false,"baseCatalogName":null,"baseColumnName":null,"baseSchemaName":null,"baseServerName":null,"baseTableName":null,"columnName":"ModifiedDate","columnOrdinal":3,"columnSize":8,"isAliased":null,"isAutoIncrement":false,"isExpression":null,"isHidden":null,"isIdentity":false,"isKey":null,"isLong":false,"isReadOnly":false,"isUnique":false,"numericPrecision":23,"numericScale":3,"udtAssemblyQualifiedName":null,"dataType":"System.DateTime, System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","dataTypeName":"datetime"}],"specialAction":{"none":true,"expectYukonXMLShowPlan":false}}],"specialAction":{"none":true,"expectYukonXMLShowPlan":false}},"ownerUri":"mismatchquerycompleteresponse_1"}}Content-Length: 3510
+
+{"jsonrpc":"2.0","method":"query/complete","params":{"ownerUri":"mismatchquerycompleteresponse_1","batchSummaries":[{"executionElapsed":"00:00:00.0805260","executionEnd":"2018-02-08T13:20:15.3449966-08:00","executionStart":"2018-02-08T13:20:15.2644706-08:00","hasError":false,"id":0,"selection":{"endColumn":39,"endLine":1,"startColumn":0,"startLine":0},"resultSetSummaries":[{"id":0,"batchId":0,"rowCount":16,"columnInfo":[{"isBytes":false,"isChars":false,"isSqlVariant":false,"isUdt":false,"isXml":false,"isJson":false,"sqlDbType":16,"isSqlXmlType":false,"isUnknownType":false,"isUpdatable":false,"allowDBNull":false,"baseCatalogName":null,"baseColumnName":null,"baseSchemaName":null,"baseServerName":null,"baseTableName":null,"columnName":"DepartmentID","columnOrdinal":0,"columnSize":2,"isAliased":null,"isAutoIncrement":true,"isExpression":null,"isHidden":null,"isIdentity":true,"isKey":null,"isLong":false,"isReadOnly":true,"isUnique":false,"numericPrecision":5,"numericScale":255,"udtAssemblyQualifiedName":null,"dataType":"System.Int16, System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","dataTypeName":"smallint"},{"isBytes":false,"isChars":true,"isSqlVariant":false,"isUdt":false,"isXml":false,"isJson":false,"sqlDbType":12,"isSqlXmlType":false,"isUnknownType":false,"isUpdatable":true,"allowDBNull":false,"baseCatalogName":null,"baseColumnName":null,"baseSchemaName":null,"baseServerName":null,"baseTableName":null,"columnName":"Name","columnOrdinal":1,"columnSize":50,"isAliased":null,"isAutoIncrement":false,"isExpression":null,"isHidden":null,"isIdentity":false,"isKey":null,"isLong":false,"isReadOnly":false,"isUnique":false,"numericPrecision":255,"numericScale":255,"udtAssemblyQualifiedName":null,"dataType":"System.String, System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","dataTypeName":"nvarchar"},{"isBytes":false,"isChars":true,"isSqlVariant":false,"isUdt":false,"isXml":false,"isJson":false,"sqlDbType":12,"isSqlXmlType":false,"isUnknownType":false,"isUpdatable":true,"allowDBNull":false,"baseCatalogName":null,"baseColumnName":null,"baseSchemaName":null,"baseServerName":null,"baseTableName":null,"columnName":"GroupName","columnOrdinal":2,"columnSize":50,"isAliased":null,"isAutoIncrement":false,"isExpression":null,"isHidden":null,"isIdentity":false,"isKey":null,"isLong":false,"isReadOnly":false,"isUnique":false,"numericPrecision":255,"numericScale":255,"udtAssemblyQualifiedName":null,"dataType":"System.String, System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","dataTypeName":"nvarchar"},{"isBytes":false,"isChars":false,"isSqlVariant":false,"isUdt":false,"isXml":false,"isJson":false,"sqlDbType":4,"isSqlXmlType":false,"isUnknownType":false,"isUpdatable":true,"allowDBNull":false,"baseCatalogName":null,"baseColumnName":null,"baseSchemaName":null,"baseServerName":null,"baseTableName":null,"columnName":"ModifiedDate","columnOrdinal":3,"columnSize":8,"isAliased":null,"isAutoIncrement":false,"isExpression":null,"isHidden":null,"isIdentity":false,"isKey":null,"isLong":false,"isReadOnly":false,"isUnique":false,"numericPrecision":23,"numericScale":3,"udtAssemblyQualifiedName":null,"dataType":"System.DateTime, System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","dataTypeName":"datetime"}],"specialAction":{"none":true,"expectYukonXMLShowPlan":false}}],"specialAction":{"none":true,"expectYukonXMLShowPlan":false}}]}}