Support portable filter expressions with AzureVectorStore

Author: tzolov

spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/vectorstore/azure/AzureVectorStoreAutoConfiguration.java

@@ -21,7 +21,7 @@
 import com.azure.search.documents.indexes.SearchIndexClientBuilder;
 
 import org.springframework.ai.embedding.EmbeddingClient;
-import org.springframework.ai.vectorstore.AzureVectorStore;
+import org.springframework.ai.vectorstore.azure.AzureVectorStore;
 import org.springframework.ai.vectorstore.VectorStore;
 import org.springframework.boot.autoconfigure.AutoConfiguration;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;

spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/vectorstore/azure/AzureVectorStoreProperties.java

@@ -16,7 +16,7 @@
 
 package org.springframework.ai.autoconfigure.vectorstore.azure;
 
-import org.springframework.ai.vectorstore.AzureVectorStore;
+import org.springframework.ai.vectorstore.azure.AzureVectorStore;
 import org.springframework.boot.context.properties.ConfigurationProperties;
 
 /**

spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vectorstore/azure/AzureVectorStoreAutoConfigurationIT.java

@@ -31,7 +31,7 @@
 import org.springframework.ai.document.Document;
 import org.springframework.ai.embedding.EmbeddingClient;
 import org.springframework.ai.embedding.TransformersEmbeddingClient;
-import org.springframework.ai.vectorstore.AzureVectorStore;
+import org.springframework.ai.vectorstore.azure.AzureVectorStore;
 import org.springframework.ai.vectorstore.SearchRequest;
 import org.springframework.ai.vectorstore.VectorStore;
 import org.springframework.boot.autoconfigure.AutoConfigurations;

vector-stores/spring-ai-azure/README.md

@@ -43,7 +43,7 @@ Add these dependencies to your project:
 1. Select an Embeddings interface implementation.
 You can choose between:
 
-* OpenAI Embedding:
+* or OpenAI Embedding:
 
    ```xml
    <dependency>
@@ -102,10 +102,21 @@ To create a vector store, you can use the following code by injecting the `Searc
 ```java
 @Bean
 public VectorStore vectorStore(SearchIndexClient searchIndexClient, EmbeddingClient embeddingClient) {
-  return new AzureVectorStore(searchIndexClient, embeddingClient);
+  return new AzureVectorStore(searchIndexClient, embeddingClient,
+    // Define the metadata fields to be used
+    // in the similarity search filters.
+    List.of(MetadataField.text("country"),
+            MetadataField.int64("year"),
+            MetadataField.bool("active")));
 }
 ```
 
+> [!NOTE]
+> You must list explicitly all metadata field names and types for any metadata key used in filter expression.
+>The list above registers filterable metadata fields: `country` of type `TEXT`, `year` of type `INT64` and `active` of type `BOOLEAN`.
+>
+> If the filterable metadata fields is expanded with new entires, you have to (re)upload/update the documents with this metadata.
+
 In your main code, create some documents
 
 ```java
@@ -124,18 +135,64 @@ vectorStore.add(List.of(document));
 And finally, retrieve documents similar to a query:
 
 ```java
-List<Document> results = vectorStore.similaritySearch(SearchRequest.query("Spring").withTopK(5));
+List<Document> results = vectorStore.similaritySearch(
+    SearchRequest
+      .query("Spring")
+      .withTopK(5));
 ```
 
 If all goes well, you should retrieve the document containing the text "Spring AI rocks!!".
 
+### Metadata filtering
+
+You can leverage the generic, portable [metadata filters](https://docs.spring.io/spring-ai/reference/api/vectordbs.html#_metadata_filters) with AzureVectorStore as well.
+
+For example you can use either the text expression language:
+
+```java
+vectorStore.similaritySearch(
+   SearchRequest
+      .query("The World")
+      .withTopK(TOP_K)
+      .withSimilarityThreshold(SIMILARITY_THRESHOLD)
+      .withFilterExpression("country in ['UK', 'NL'] && year >= 2020"));
+```
+
+or programmatically using the expression DSL:
+
+```java
+FilterExpressionBuilder b = Filter.builder();
+
+vectorStore.similaritySearch(
+    SearchRequest
+      .query("The World")
+      .withTopK(TOP_K)
+      .withSimilarityThreshold(SIMILARITY_THRESHOLD)
+      .withFilterExpression(b.and(
+         b.in("country", "UK", "NL"),
+         b.gte("year", 2020)).build()));
+```
+
+The, portable, filter expressions get automatically converted into the proprietary Azure Search [OData filters](https://learn.microsoft.com/en-us/azure/search/search-query-odata-filter).
+For example the following, portable, filter expression
+
+```sql
+country in ['UK', 'NL'] && year >= 2020
+```
+
+is converted into Azure, OData, [filter expression](https://learn.microsoft.com/en-us/azure/search/search-query-odata-filter):
+
+```graphQL
+$filter search.in(meta_country, 'UK,NL', ',') and meta_year ge 2020
+```
+
 ## Integration With Azure OpenAI Studio Data Ingestion
 
 Azure Open AI services provides a convenient method to upload documents into an Index as described in this Microsoft
 [learning document](https://learn.microsoft.com/en-us/azure/ai-services/openai/use-your-data-quickstart?tabs=command-line&pivots=programming-language-csharp).
 The `AzureVectorStore` implementation is compatible with indexes that use this methodology facilitating an *easier* way to integrate with your existing documents for the purpose of searching and integrating with the AI system.
 
-## <a name="appendix_a" /> Appendix A: Create Vector Store Search Index
+## <a name="appendix_a" /> Appendix A: Create Vector Store Search Index </a>
 
 The easiest way to crate a search index manually, is to create one from a JSON document.
 This can be done by clicking on the `Indexes` link under the `Search management` section.

vector-stores/spring-ai-azure/src/main/java/org/springframework/ai/vectorstore/azure/AzureAiSearchFilterExpressionConverter.java

@@ -0,0 +1,186 @@
+/*
+ * Copyright 2023-2023 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.vectorstore.azure;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.TimeZone;
+import java.util.regex.Pattern;
+
+import org.springframework.ai.vectorstore.azure.AzureVectorStore.MetadataField;
+import org.springframework.ai.vectorstore.filter.Filter;
+import org.springframework.ai.vectorstore.filter.Filter.Expression;
+import org.springframework.ai.vectorstore.filter.Filter.ExpressionType;
+import org.springframework.ai.vectorstore.filter.Filter.Group;
+import org.springframework.ai.vectorstore.filter.Filter.Key;
+import org.springframework.ai.vectorstore.filter.converter.AbstractFilterExpressionConverter;
+import org.springframework.util.Assert;
+
+/**
+ * Converts {@link Expression} into Azure Search OData filter syntax.
+ * https://learn.microsoft.com/en-us/azure/search/search-query-odata-filter
+ *
+ * @author Christian Tzolov
+ */
+public class AzureAiSearchFilterExpressionConverter extends AbstractFilterExpressionConverter {
+
+	private static Pattern DATE_FORMAT_PATTERN = Pattern.compile("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z");
+
+	private final List<MetadataField> filterMetadataFields;
+
+	private final SimpleDateFormat dateFormat;
+
+	private List<String> allowedIdentifierNames;
+
+	public AzureAiSearchFilterExpressionConverter(List<MetadataField> filterMetadataFields) {
+		Assert.notNull(filterMetadataFields, "The filterMetadataFields can not null.");
+
+		this.allowedIdentifierNames = filterMetadataFields.stream().map(MetadataField::name).toList();
+		this.filterMetadataFields = filterMetadataFields;
+		this.dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
+		// this.dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZZZZZ");
+		this.dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+	}
+
+	@Override
+	protected void doExpression(Expression expression, StringBuilder context) {
+		if (expression.type() == ExpressionType.IN || expression.type() == ExpressionType.NIN) {
+			context.append(getOperationSymbol(expression));
+			context.append("(");
+			this.convertOperand(expression.left(), context);
+			context.append(", ");
+			this.convertOperand(expression.right(), context);
+			context.append(", ',')");
+		}
+		else {
+			this.convertOperand(expression.left(), context);
+			context.append(getOperationSymbol(expression));
+			this.convertOperand(expression.right(), context);
+		}
+	}
+
+	protected void doStartValueRange(Filter.Value listValue, StringBuilder context) {
+		context.append("'");
+	}
+
+	protected void doEndValueRange(Filter.Value listValue, StringBuilder context) {
+		context.append("'");
+	}
+
+	private String getOperationSymbol(Expression exp) {
+		switch (exp.type()) {
+			case AND:
+				return " and ";
+			case OR:
+				return " or ";
+			case EQ:
+				return " eq ";
+			case NE:
+				return " ne ";
+			case LT:
+				return " lt ";
+			case LTE:
+				return " le ";
+			case GT:
+				return " gt ";
+			case GTE:
+				return " ge ";
+			case IN:
+				return " search.in";
+			case NIN:
+				return " not search.in";
+			default:
+				throw new RuntimeException("Not supported expression type: " + exp.type());
+		}
+	}
+
+	@Override
+	public void doKey(Key key, StringBuilder context) {
+		var hasOuterQuotes = hasOuterQuotes(key.key());
+		var identifier = (hasOuterQuotes) ? removeOuterQuotes(key.key()) : key.key();
+		var prefixedIdentifier = withMetaPrefix(identifier);
+		if (hasOuterQuotes) {
+			prefixedIdentifier = "'" + prefixedIdentifier.trim() + "'";
+		}
+		context.append(prefixedIdentifier);
+	}
+
+	public String withMetaPrefix(String identifier) {
+
+		if (this.allowedIdentifierNames.contains(identifier)) {
+			return "meta_" + identifier;
+		}
+
+		throw new IllegalArgumentException("Not allowed filter identifier name: " + identifier);
+	}
+
+	@Override
+	protected void doValue(Filter.Value filterValue, StringBuilder context) {
+		if (filterValue.value() instanceof List list) {
+			doStartValueRange(filterValue, context);
+			int c = 0;
+			for (Object v : list) {
+				// this.doSingleValue(v, context);
+				context.append(v);
+				if (c++ < list.size() - 1) {
+					this.doAddValueRangeSpitter(filterValue, context);
+				}
+			}
+			this.doEndValueRange(filterValue, context);
+		}
+		else {
+			this.doSingleValue(filterValue.value(), context);
+		}
+	}
+
+	@Override
+	protected void doSingleValue(Object value, StringBuilder context) {
+		if (value instanceof Date date) {
+			context.append(this.dateFormat.format(date));
+		}
+		else if (value instanceof String text) {
+			if (DATE_FORMAT_PATTERN.matcher(text).matches()) {
+				try {
+					Date date = this.dateFormat.parse(text);
+					context.append(this.dateFormat.format(date));
+				}
+				catch (ParseException e) {
+					throw new IllegalArgumentException("Invalid date type:" + text, e);
+				}
+			}
+			else {
+				context.append(String.format("'%s'", text));
+			}
+		}
+		else {
+			context.append(value);
+		}
+	}
+
+	@Override
+	public void doStartGroup(Group group, StringBuilder context) {
+		context.append("(");
+	}
+
+	@Override
+	public void doEndGroup(Group group, StringBuilder context) {
+		context.append(")");
+	}
+
+}