Skip to content

Commit faee5fe

Browse files
tzolovmarkpollack
tzolov
authored and
markpollack
committed
Add support for Weaviate Vector Store
 - Add Weaviate Vector Store implementation. - Implement a converter of portable Filter.Expressions into native, Weaviate GraphQL Were expressions. - Support for Weaviater schema auto-registration of filtarable metadata fields. - Add auto-configration, spring properties and tests. - WeaviateVectorStore ITs. - Add README.md Resolves #100
1 parent 332c92a commit faee5fe

File tree

14 files changed

+1999
-0
lines changed

14 files changed

+1999
-0
lines changed

‎pom.xml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
<module>vector-stores/spring-ai-pinecone</module>
3636
<module>vector-stores/spring-ai-chroma</module>
3737
<module>vector-stores/spring-ai-azure</module>
38+
<module>vector-stores/spring-ai-weaviate</module>
3839

3940
</modules>
4041

@@ -96,6 +97,7 @@
9697
<protobuf-java-util.version>3.24.4</protobuf-java-util.version>
9798
<fastjson.version>2.0.42</fastjson.version>
9899
<azure-search.version>11.6.0</azure-search.version>
100+
<weaviate-client.version>4.4.1</weaviate-client.version>
99101

100102
<!-- testing dependencies -->
101103
<testcontainers.version>1.19.0</testcontainers.version>

‎spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs.adoc

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,8 @@ These are the available implementations of the `VectorStore` interface:
9393
* PgVector [`PgVectorStore`]: The https://github.com/pgvector/pgvector[PostgreSQL/PGVector] vector store.
9494
* Milvus [`MilvusVectorStore`]: The https://milvus.io/[Milvus] vector store
9595
* Neo4j [`Neo4jVectorStore`]: The https://neo4j.com/[Neo4j] vector store
96+
* Weaviate [`WeaviateVectorStore`] The https://weaviate.io/[Weaviate] vector store
97+
* Azure Vector Search [`AzureVectorStore`] the https://learn.microsoft.com/en-us/azure/search/vector-search-overview[Azure] vector store
9698

9799
More implementations may be supported in future releases.
98100

‎spring-ai-spring-boot-autoconfigure/pom.xml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,14 @@
117117
<optional>true</optional>
118118
</dependency>
119119

120+
<!-- Weaviate Vector Store -->
121+
<dependency>
122+
<groupId>org.springframework.experimental.ai</groupId>
123+
<artifactId>spring-ai-weaviate-store</artifactId>
124+
<version>${project.parent.version}</version>
125+
<optional>true</optional>
126+
</dependency>
127+
120128

121129
<dependency>
122130
<groupId>org.springframework.boot</groupId>

‎spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/vectorstore/weaviate/WeaviateVectorStoreAutoConfiguration.java

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
/*
2+
* Copyright 2023-2023 the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.springframework.ai.autoconfigure.vectorstore.weaviate;
18+
19+
import org.springframework.ai.embedding.EmbeddingClient;
20+
import org.springframework.ai.vectorstore.VectorStore;
21+
import org.springframework.ai.vectorstore.WeaviateVectorStore;
22+
import org.springframework.ai.vectorstore.WeaviateVectorStore.WeaviateVectorStoreConfig;
23+
import org.springframework.ai.vectorstore.WeaviateVectorStore.WeaviateVectorStoreConfig.MetadataField;
24+
import org.springframework.boot.autoconfigure.AutoConfiguration;
25+
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
26+
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
27+
import org.springframework.boot.context.properties.EnableConfigurationProperties;
28+
import org.springframework.context.annotation.Bean;
29+
30+
/**
31+
* @author Christian Tzolov
32+
*/
33+
@AutoConfiguration
34+
@ConditionalOnClass({ EmbeddingClient.class, WeaviateVectorStore.class })
35+
@EnableConfigurationProperties({ WeaviateVectorStoreProperties.class })
36+
public class WeaviateVectorStoreAutoConfiguration {
37+
38+
@Bean
39+
@ConditionalOnMissingBean
40+
public VectorStore vectorStore(EmbeddingClient embeddingClient, WeaviateVectorStoreProperties properties) {
41+
42+
WeaviateVectorStoreConfig.Builder configBuilder = WeaviateVectorStore.WeaviateVectorStoreConfig.builder()
43+
.withScheme(properties.getScheme())
44+
.withApiKey(properties.getApiKey())
45+
.withHost(properties.getHost())
46+
.withHeaders(properties.getHeaders())
47+
.withObjectClass(properties.getObjectClass())
48+
.withFilterableMetadataFields(properties.getFilterField()
49+
.entrySet()
50+
.stream()
51+
.map(e -> new MetadataField(e.getKey(), e.getValue()))
52+
.toList())
53+
.withConsistencyLevel(properties.getConsistencyLevel());
54+
55+
return new WeaviateVectorStore(configBuilder.build(), embeddingClient);
56+
}
57+
58+
}

‎spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/vectorstore/weaviate/WeaviateVectorStoreProperties.java

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
/*
2+
* Copyright 2023-2023 the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.springframework.ai.autoconfigure.vectorstore.weaviate;
18+
19+
import java.util.Map;
20+
21+
import org.springframework.ai.vectorstore.WeaviateVectorStore.WeaviateVectorStoreConfig;
22+
import org.springframework.ai.vectorstore.WeaviateVectorStore.WeaviateVectorStoreConfig.ConsistentLevel;
23+
import org.springframework.ai.vectorstore.WeaviateVectorStore.WeaviateVectorStoreConfig.MetadataField;
24+
import org.springframework.boot.context.properties.ConfigurationProperties;
25+
26+
/**
27+
* @author Christian Tzolov
28+
*/
29+
@ConfigurationProperties(WeaviateVectorStoreProperties.CONFIG_PREFIX)
30+
public class WeaviateVectorStoreProperties {
31+
32+
public static final String CONFIG_PREFIX = "spring.ai.vectorstore.weaviate";
33+
34+
private String scheme = "http";
35+
36+
private String host = "localhost:8080";
37+
38+
private String apiKey = "";
39+
40+
private String objectClass = "SpringAiWeaviate";
41+
42+
private ConsistentLevel consistencyLevel = WeaviateVectorStoreConfig.ConsistentLevel.ONE;
43+
44+
/**
45+
* spring.ai.vectorstore.weaviate.filter-field.<field-name>=<field-type>
46+
*/
47+
private Map<String, MetadataField.Type> filterField = Map.of();
48+
49+
private Map<String, String> headers = Map.of();
50+
51+
public void setScheme(String scheme) {
52+
this.scheme = scheme;
53+
}
54+
55+
public String getScheme() {
56+
return scheme;
57+
}
58+
59+
public void setHost(String host) {
60+
this.host = host;
61+
}
62+
63+
public String getHost() {
64+
return host;
65+
}
66+
67+
public String getApiKey() {
68+
return apiKey;
69+
}
70+
71+
public void setApiKey(String apiKey) {
72+
this.apiKey = apiKey;
73+
}
74+
75+
public String getObjectClass() {
76+
return objectClass;
77+
}
78+
79+
public void setObjectClass(String indexName) {
80+
this.objectClass = indexName;
81+
}
82+
83+
public ConsistentLevel getConsistencyLevel() {
84+
return consistencyLevel;
85+
}
86+
87+
public void setConsistencyLevel(ConsistentLevel consistencyLevel) {
88+
this.consistencyLevel = consistencyLevel;
89+
}
90+
91+
public Map<String, String> getHeaders() {
92+
return headers;
93+
}
94+
95+
public void setHeaders(Map<String, String> headers) {
96+
this.headers = headers;
97+
}
98+
99+
public Map<String, MetadataField.Type> getFilterField() {
100+
return filterField;
101+
}
102+
103+
public void setFilterField(Map<String, MetadataField.Type> filterMetadataFields) {
104+
this.filterField = filterMetadataFields;
105+
}
106+
107+
}

‎spring-ai-spring-boot-autoconfigure/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,4 @@ org.springframework.ai.autoconfigure.embedding.transformer.TransformersEmbedding
88
org.springframework.ai.autoconfigure.huggingface.HuggingfaceAutoConfiguration
99
org.springframework.ai.autoconfigure.vectorstore.chroma.ChromaVectorStoreAutoConfiguration
1010
org.springframework.ai.autoconfigure.vectorstore.azure.AzureVectorStoreAutoConfiguration
11+
org.springframework.ai.autoconfigure.vectorstore.weaviate.WeaviateVectorStoreAutoConfiguration

‎spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/vectorstore/weaviate/WeaviateVectorStoreAutoConfigurationTests.java

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
/*
2+
* Copyright 2023-2023 the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.springframework.ai.autoconfigure.vectorstore.weaviate;
18+
19+
import java.util.List;
20+
import java.util.Map;
21+
22+
import org.junit.jupiter.api.Test;
23+
import org.testcontainers.containers.GenericContainer;
24+
import org.testcontainers.junit.jupiter.Container;
25+
import org.testcontainers.junit.jupiter.Testcontainers;
26+
27+
import org.springframework.ai.document.Document;
28+
import org.springframework.ai.embedding.EmbeddingClient;
29+
import org.springframework.ai.embedding.TransformersEmbeddingClient;
30+
import org.springframework.ai.vectorstore.SearchRequest;
31+
import org.springframework.ai.vectorstore.VectorStore;
32+
import org.springframework.ai.vectorstore.WeaviateVectorStore.WeaviateVectorStoreConfig.MetadataField;
33+
import org.springframework.boot.autoconfigure.AutoConfigurations;
34+
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
35+
import org.springframework.context.annotation.Bean;
36+
import org.springframework.context.annotation.Configuration;
37+
38+
import static org.assertj.core.api.Assertions.assertThat;
39+
40+
/**
41+
* @author Christian Tzolov
42+
*/
43+
@Testcontainers
44+
public class WeaviateVectorStoreAutoConfigurationTests {
45+
46+
@Container
47+
static GenericContainer<?> weaviateContainer = new GenericContainer<>("semitechnologies/weaviate:1.22.4")
48+
.withEnv("AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED", "true")
49+
.withEnv("PERSISTENCE_DATA_PATH", "/var/lib/weaviate")
50+
.withEnv("QUERY_DEFAULTS_LIMIT", "25")
51+
.withEnv("DEFAULT_VECTORIZER_MODULE", "none")
52+
.withEnv("CLUSTER_HOSTNAME", "node1")
53+
.withExposedPorts(8080);
54+
55+
private final ApplicationContextRunner contextRunner = new ApplicationContextRunner()
56+
.withConfiguration(AutoConfigurations.of(WeaviateVectorStoreAutoConfiguration.class))
57+
.withUserConfiguration(Config.class)
58+
.withPropertyValues("spring.ai.vectorstore.weaviate.scheme=http",
59+
"spring.ai.vectorstore.weaviate.host=localhost:" + weaviateContainer.getMappedPort(8080),
60+
"spring.ai.vectorstore.weaviate.filter-field.country=TEXT",
61+
"spring.ai.vectorstore.weaviate.filter-field.year=NUMBER",
62+
"spring.ai.vectorstore.weaviate.filter-field.active=BOOLEAN",
63+
"spring.ai.vectorstore.weaviate.filter-field.price=NUMBER");
64+
65+
@Test
66+
public void addAndSearchWithFilters() {
67+
68+
contextRunner.run(context -> {
69+
70+
WeaviateVectorStoreProperties properties = context.getBean(WeaviateVectorStoreProperties.class);
71+
72+
assertThat(properties.getFilterField()).hasSize(4);
73+
74+
assertThat(properties.getFilterField().get("country")).isEqualTo(MetadataField.Type.TEXT);
75+
assertThat(properties.getFilterField().get("year")).isEqualTo(MetadataField.Type.NUMBER);
76+
assertThat(properties.getFilterField().get("active")).isEqualTo(MetadataField.Type.BOOLEAN);
77+
assertThat(properties.getFilterField().get("price")).isEqualTo(MetadataField.Type.NUMBER);
78+
79+
VectorStore vectorStore = context.getBean(VectorStore.class);
80+
81+
var bgDocument = new Document("The World is Big and Salvation Lurks Around the Corner",
82+
Map.of("country", "Bulgaria", "price", 3.14, "active", true, "year", 2020));
83+
var nlDocument = new Document("The World is Big and Salvation Lurks Around the Corner",
84+
Map.of("country", "Netherland", "price", 1.57, "active", false, "year", 2023));
85+
86+
vectorStore.add(List.of(bgDocument, nlDocument));
87+
88+
var request = SearchRequest.query("The World").withTopK(5);
89+
90+
List<Document> results = vectorStore.similaritySearch(request);
91+
assertThat(results).hasSize(2);
92+
93+
results = vectorStore
94+
.similaritySearch(request.withSimilarityThresholdAll().withFilterExpression("country == 'Bulgaria'"));
95+
assertThat(results).hasSize(1);
96+
assertThat(results.get(0).getId()).isEqualTo(bgDocument.getId());
97+
98+
results = vectorStore
99+
.similaritySearch(request.withSimilarityThresholdAll().withFilterExpression("country == 'Netherland'"));
100+
assertThat(results).hasSize(1);
101+
assertThat(results.get(0).getId()).isEqualTo(nlDocument.getId());
102+
103+
results = vectorStore.similaritySearch(
104+
request.withSimilarityThresholdAll().withFilterExpression("price > 1.57 && active == true"));
105+
assertThat(results).hasSize(1);
106+
assertThat(results.get(0).getId()).isEqualTo(bgDocument.getId());
107+
108+
results = vectorStore
109+
.similaritySearch(request.withSimilarityThresholdAll().withFilterExpression("year in [2020, 2023]"));
110+
assertThat(results).hasSize(2);
111+
112+
results = vectorStore.similaritySearch(
113+
request.withSimilarityThresholdAll().withFilterExpression("year > 2020 && year <= 2023"));
114+
assertThat(results).hasSize(1);
115+
assertThat(results.get(0).getId()).isEqualTo(nlDocument.getId());
116+
117+
// Remove all documents from the store
118+
vectorStore.delete(List.of(bgDocument, nlDocument).stream().map(doc -> doc.getId()).toList());
119+
});
120+
}
121+
122+
@Configuration(proxyBeanMethods = false)
123+
static class Config {
124+
125+
@Bean
126+
public EmbeddingClient embeddingClient() {
127+
return new TransformersEmbeddingClient();
128+
}
129+
130+
}
131+
132+
}

0 commit comments

Comments
 (0)