New 'UNLIST' function (#8418)

Implementation of the new internal function UNLIST

Author: ChudaykinAlex

builds/win32/msvc15/engine_static.vcxproj

@@ -140,6 +140,7 @@
     <ClCompile Include="..\..\..\src\jrd\recsrc\SingularStream.cpp" />
     <ClCompile Include="..\..\..\src\jrd\recsrc\SkipRowsStream.cpp" />
     <ClCompile Include="..\..\..\src\jrd\recsrc\SortedStream.cpp" />
+	<ClCompile Include="..\..\..\src\jrd\recsrc\TableValueFunctionScan.cpp" />
     <ClCompile Include="..\..\..\src\jrd\recsrc\Union.cpp" />
     <ClCompile Include="..\..\..\src\jrd\recsrc\VirtualTableScan.cpp" />
     <ClCompile Include="..\..\..\src\jrd\recsrc\WindowedStream.cpp" />

builds/win32/msvc15/engine_static.vcxproj.filters

@@ -111,7 +111,10 @@
     <ClCompile Include="..\..\..\src\jrd\recsrc\SortedStream.cpp">
       <Filter>JRD files\Data Access</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\..\src\jrd\recsrc\Union.cpp">
+	<ClCompile Include="..\..\..\src\jrd\recsrc\Union.cpp">
+      <Filter>JRD files\Data Access</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\..\src\jrd\recsrc\TableValueFunctionScan.cpp">
       <Filter>JRD files\Data Access</Filter>
     </ClCompile>
     <ClCompile Include="..\..\..\src\jrd\recsrc\VirtualTableScan.cpp">

doc/sql.extensions/README.unlist

@@ -0,0 +1,71 @@
+SQL Language Extension: UNLIST
+
+Function:
+	The function parses the input string using the specified delimiter (comma "," is implied by default) and returns the identified substrings as discrete records containing a single field. Additionally, the desired type of the returned field can be specified. If the specified data type conversion is impossible, an error is raised at runtime.
+
+Author:
+	Chudaykin Alex <[email protected]>
+
+Format
+	<table value function> ::=
+		UNLIST ( <input> [, <separator>] [, <data type conversion>] ) [AS] <correlation name> [ ( <derived column name> ) ]
+
+		<input> ::= <value>
+
+		<separator> ::= <value>
+
+		<data type conversion> ::= RETURNING <data type>
+
+Syntax Rules:
+
+1) <input>: any value expression that returns a string/blob of characters (or may be converted to a string), including string literals, table columns, constants, variables, expressions, etc. This parameter is mandatory.
+2) <separator>: optional value expression that returns a string which is used as a delimiter (i.e. it separates one value from another inside the input string). It may also be a BLOB TEXT value, limited to 32KB. If an empty string is specified, the output will be just one record containing the input string. If omitted, the comma character is used as a delimiter.
+2) <data type>: target data type to convert the output values into. Alternatively, a domain can be specified as the returned type. If omitted, VARCHAR(32) is implied. Feel free to suggest any better alternative default.
+3) <correlation name>: alias of the record set returned by the UNLIST function. It is a mandatory parameter (per SQL standard).
+4) <derived column name>: optional alias of the column returned by the UNLIST function. If omitted, UNLIST is used as an alias.
+
+Example:
+A)
+	SELECT * FROM UNLIST('1,2,3,4,5') AS U;
+
+B)
+	SELECT * FROM UNLIST('100:200:300:400:500', ':' RETURNING INT) AS U;
+
+C)
+	SELECT U.* FROM UNLIST(‘text1, text2, text3’) AS U;
+
+D)
+	SELECT C0 FROM UNLIST(‘text1, text2, text3’) AS U(C0);
+
+E)
+	SELECT U.C0 FROM UNLIST(‘text1, text2, text3’) AS U(C0);
+
+F)
+	SET TERM ^ ;
+	RECREATE PROCEDURE TEST_PROC RETURNS (PROC_RETURN_INT INT)
+	AS
+	DECLARE VARIABLE text VARCHAR(11);
+	BEGIN
+		text = '123:123:123';
+		FOR SELECT * FROM UNLIST( :text, ':' RETURNING INT) AS A INTO :PROC_RETURN_INT DO
+		SUSPEND;
+	END^
+	SET TERM ; ^
+	SELECT * FROM TEST_PROC;
+
+G)
+	CREATE DOMAIN D1 AS INT;
+	SELECT TEST_DOMAIN FROM UNLIST('1,2,3,4' RETURNING D1) AS A(TEST_DOMAIN);
+
+	CREATE TABLE TABLE_TEST (COL1 INT);
+	SELECT TEST_TYPE_OF FROM UNLIST('1,2,3,4' RETURNING TYPE OF COLUMN TABLE_TEST.COL1) AS A(TEST_TYPE_OF);
+H)
+	CREATE VIEW TEST_VIEW AS SELECT * FROM UNLIST('1,2,3,4') AS A(B);
+	SELECT B FROM TEST_VIEW;
+
+Unacceptable behavior:
+	SELECT UNLIST FROM UNLIST('UNLIST,A,S,A') AS A;
+
+
+
+

src/common/ParserTokens.h

@@ -524,6 +524,7 @@ PARSER_TOKEN(TOK_UNICODE_CHAR, "UNICODE_CHAR", true)
 PARSER_TOKEN(TOK_UNICODE_VAL, "UNICODE_VAL", true)
 PARSER_TOKEN(TOK_UNION, "UNION", false)
 PARSER_TOKEN(TOK_UNIQUE, "UNIQUE", false)
+PARSER_TOKEN(TOK_UNLIST, "UNLIST", true)
 PARSER_TOKEN(TOK_UNKNOWN, "UNKNOWN", false)
 PARSER_TOKEN(TOK_UPDATE, "UPDATE", false)
 PARSER_TOKEN(TOK_UPDATING, "UPDATING", false)

src/dsql/DdlNodes.epp

@@ -9119,6 +9119,14 @@ void CreateAlterViewNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScra
 		else
 			updatable = false;
 
+
+		if (field && context && (context->ctx_flags & CTX_blr_fields))
+		{
+			field = nullptr;
+			context = nullptr;
+			updatable = false;
+		}
+
 		// If this is an expression, check to make sure there is a name specified.
 
 		if (!ptr && !fieldStr)

src/dsql/DsqlCompilerScratch.cpp

@@ -945,6 +945,11 @@ bool DsqlCompilerScratch::pass1RelProcIsRecursive(RecordSourceNode* input)
 		relName = relNode->dsqlName;
 		relAlias = relNode->alias;
 	}
+	else if (auto tableValueFunctionNode = nodeAs<TableValueFunctionSourceNode>(input))
+	{
+		relName = tableValueFunctionNode->dsqlName;
+		relAlias = tableValueFunctionNode->alias.c_str();
+	}
 	//// TODO: LocalTableSourceNode
 	else
 		return false;

src/dsql/ExprNodes.cpp

@@ -6027,6 +6027,11 @@ DmlNode* FieldNode::parse(thread_db* tdbb, MemoryPool& pool, CompilerScratch* cs
 					Arg::Str(name) << Arg::Str(procedure->getName().toString()));
 			}
 		}
+		else if (tail->csb_table_value_fun)
+		{
+			csb->csb_blr_reader.getMetaName(name);
+			id = tail->csb_table_value_fun->getId(name);
+		}
 		else
 		{
 			jrd_rel* relation = tail->csb_relation;
@@ -6250,6 +6255,13 @@ ValueExprNode* FieldNode::internalDsqlPass(DsqlCompilerScratch* dsqlScratch, Rec
 							procNode->dsqlContext = stackContext;
 							*list = procNode;
 						}
+						else if (context->ctx_table_value_fun)
+						{
+							auto tableValueFunctionNode = FB_NEW_POOL(*tdbb->getDefaultPool())
+								TableValueFunctionSourceNode(*tdbb->getDefaultPool());
+							tableValueFunctionNode->dsqlContext = stackContext;
+							*list = tableValueFunctionNode;
+						}
 						//// TODO: LocalTableSourceNode
 
 						fb_assert(*list);
@@ -6446,9 +6458,28 @@ dsql_fld* FieldNode::resolveContext(DsqlCompilerScratch* dsqlScratch, const Meta
 		return nullptr;
 	}
 
+	const TEXT* dsqlName = nullptr;
+	dsql_fld* outputField = nullptr;
 	dsql_rel* relation = context->ctx_relation;
 	dsql_prc* procedure = context->ctx_procedure;
-	if (!relation && !procedure)
+	dsql_tab_func* tableValueFunctionContext = context->ctx_table_value_fun;
+
+	if (relation)
+	{
+		dsqlName = relation->rel_name.c_str();
+		outputField = relation->rel_fields;
+	}
+	else if (procedure)
+	{
+		dsqlName = procedure->prc_name.identifier.c_str();
+		outputField = procedure->prc_outputs;
+	}
+	else if	(tableValueFunctionContext)
+	{
+		dsqlName = tableValueFunctionContext->funName.c_str();
+		outputField = tableValueFunctionContext->outputField;
+	}
+	else
 		return nullptr;
 
 	// if there is no qualifier, then we cannot match against
@@ -6491,7 +6522,9 @@ dsql_fld* FieldNode::resolveContext(DsqlCompilerScratch* dsqlScratch, const Meta
 	}
 
 	if (aliasName.isEmpty())
-		aliasName = relation ? relation->rel_name : procedure->prc_name.identifier;
+	{
+		aliasName = dsqlName;
+	}
 
 	fb_assert(aliasName.hasData());
 
@@ -6501,7 +6534,7 @@ dsql_fld* FieldNode::resolveContext(DsqlCompilerScratch* dsqlScratch, const Meta
 
 	// Lookup field in relation or procedure
 
-	return relation ? relation->rel_fields : procedure->prc_outputs;
+	return outputField;
 }
 
 bool FieldNode::dsqlAggregateFinder(AggregateFinder& visitor)

src/dsql/Nodes.h

@@ -521,6 +521,7 @@ class ExprNode : public DmlNode
 		TYPE_SELECT_EXPR,
 		TYPE_UNION,
 		TYPE_WINDOW,
+		TYPE_TABLE_VALUE_FUNCTION,
 
 		// List types
 		TYPE_REC_SOURCE_LIST,

src/dsql/StmtNodes.cpp

@@ -10682,6 +10682,8 @@ static dsql_ctx* dsqlGetContext(const RecordSourceNode* node)
 		return procNode->dsqlContext;
 	else if (auto relNode = nodeAs<RelationSourceNode>(node))
 		return relNode->dsqlContext;
+	else if (auto tableValueFunctionNode = nodeAs<TableValueFunctionSourceNode>(node))
+		return tableValueFunctionNode->dsqlContext;
 	//// TODO: LocalTableSourceNode
 	else if (auto rseNode = nodeAs<RseNode>(node))
 		return rseNode->dsqlContext;
@@ -10699,6 +10701,8 @@ static void dsqlGetContexts(DsqlContextStack& contexts, const RecordSourceNode*
 		contexts.push(procNode->dsqlContext);
 	else if (auto relNode = nodeAs<RelationSourceNode>(node))
 		contexts.push(relNode->dsqlContext);
+	else if (auto tableValueFunctionNode = nodeAs<TableValueFunctionSourceNode>(node))
+		contexts.push(tableValueFunctionNode->dsqlContext);
 	//// TODO: LocalTableSourceNode
 	else if (auto rseNode = nodeAs<RseNode>(node))
 	{

src/dsql/dsql.h

@@ -410,6 +410,21 @@ class dsql_intlsym : public pool_alloc<dsql_type_intlsym>
 	USHORT intlsym_bytes_per_char = 0;
 };
 
+
+// Table value function
+class dsql_tab_func : public pool_alloc<dsql_type_tab_func>
+{
+public:
+	explicit dsql_tab_func(MemoryPool& p)
+		: funName(p),
+		  outputField(nullptr)
+	{
+	}
+
+	MetaName	funName;			// Name of function
+	dsql_fld*	outputField;		// Output parameters
+};
+
 // values used in intlsym_flags
 
 enum intlsym_flags_vals {
@@ -454,6 +469,7 @@ class dsql_ctx : public pool_alloc<dsql_type_ctx>
 
 	dsql_rel* ctx_relation = nullptr;			// Relation for context
 	dsql_prc* ctx_procedure = nullptr;			// Procedure for context
+	dsql_tab_func* ctx_table_value_fun = nullptr;	// Table value function context
 	NestConst<ValueListNode> ctx_proc_inputs;	// Procedure input parameters
 	dsql_map* ctx_map = nullptr;				// Maps for aggregates and unions
 	RseNode* ctx_rse = nullptr;					// Sub-rse for aggregates
@@ -475,6 +491,7 @@ class dsql_ctx : public pool_alloc<dsql_type_ctx>
 	{
 		ctx_relation = v.ctx_relation;
 		ctx_procedure = v.ctx_procedure;
+		ctx_table_value_fun = v.ctx_table_value_fun;
 		ctx_proc_inputs = v.ctx_proc_inputs;
 		ctx_map = v.ctx_map;
 		ctx_rse = v.ctx_rse;
@@ -518,6 +535,7 @@ const USHORT CTX_view_with_check_store	= 0x20;		// Context of WITH CHECK OPTION
 const USHORT CTX_view_with_check_modify	= 0x40;		// Context of WITH CHECK OPTION view's modify trigger
 const USHORT CTX_cursor					= 0x80;		// Context is a cursor
 const USHORT CTX_lateral				= 0x100;	// Context is a lateral derived table
+const USHORT CTX_blr_fields				= 0x200;	// Fields of the context are defined inside BLR
 
 //! Aggregate/union map block to map virtual fields to their base
 //! TMN: NOTE! This datatype should definitely be renamed!

src/dsql/make.cpp

@@ -423,6 +423,42 @@ FieldNode* MAKE_field(dsql_ctx* context, dsql_fld* field, ValueListNode* indices
 }
 
 
+/**
+
+	MAKE_field
+
+	@brief	Make up a dsql_fld from descriptor.
+
+
+	@param field
+	@param desc
+
+ **/
+void MAKE_field(dsql_fld* field, const dsc* desc)
+{
+	DEV_BLKCHK(field, dsql_type_fld);
+
+	field->dtype = desc->dsc_dtype;
+	field->scale = desc->dsc_scale;
+	field->subType = desc->dsc_sub_type;
+	field->length = desc->dsc_length;
+
+	if (desc->dsc_dtype <= dtype_any_text)
+	{
+		field->collationId = DSC_GET_COLLATE(desc);
+		field->charSetId = DSC_GET_CHARSET(desc);
+	}
+	else if (desc->dsc_dtype == dtype_blob)
+	{
+		field->charSetId = desc->dsc_scale;
+		field->collationId = desc->dsc_flags >> 8;
+	}
+
+	if (desc->dsc_flags & DSC_nullable)
+		field->flags |= FLD_nullable;
+}
+
+
 /**
 
  	MAKE_field_name

src/dsql/make_proto.h

@@ -84,6 +84,7 @@ Jrd::LiteralNode* MAKE_const_sint64(SINT64 value, SCHAR scale);
 Jrd::ValueExprNode* MAKE_constant(const char*, Jrd::dsql_constant_type, SSHORT = 0);
 Jrd::LiteralNode* MAKE_str_constant(const Jrd::IntlString*, SSHORT);
 Jrd::FieldNode* MAKE_field(Jrd::dsql_ctx*, Jrd::dsql_fld*, Jrd::ValueListNode*);
+void MAKE_field(Jrd::dsql_fld*, const dsc*);
 Jrd::FieldNode* MAKE_field_name(const char*);
 Jrd::dsql_par* MAKE_parameter(Jrd::dsql_msg*, bool, bool, USHORT, const Jrd::ValueExprNode*);
 void MAKE_parameter_names(Jrd::dsql_par*, const Jrd::ValueExprNode*);

src/dsql/parse-conflicts.txt

@@ -1 +1 @@
-117 shift/reduce conflicts, 22 reduce/reduce conflicts.
+118 shift/reduce conflicts, 22 reduce/reduce conflicts.