mirror of
https://github.com/qdrant/landing_page.git
synced 2026-10-09 21:08:31 +02:00
Update Hybrid Search with Reranking tutorial (#2274)
* Update for Cloud Inference and data ingestion * Fix link * Review feedback * Make code snippets testable * Add C# code snippets * Add Go code snippets * Add Java code snippets * Add Rust code snippets * Add TS code snippets * Move CSV streaming/parsing to separate function
This commit is contained in:
Abdon Pijpelink
+328
@@ -0,0 +1,328 @@
|
||||
package com.example.snippets_amalgamation;
|
||||
|
||||
import static io.qdrant.client.QueryFactory.nearest;
|
||||
import static io.qdrant.client.ValueFactory.value;
|
||||
import static io.qdrant.client.VectorFactory.vector;
|
||||
import static io.qdrant.client.VectorsFactory.namedVectors;
|
||||
import static io.qdrant.client.WithPayloadSelectorFactory.enable;
|
||||
|
||||
import io.qdrant.client.QdrantClient;
|
||||
import io.qdrant.client.QdrantGrpcClient;
|
||||
import io.qdrant.client.grpc.Collections.CreateCollection;
|
||||
import io.qdrant.client.grpc.Collections.Distance;
|
||||
import io.qdrant.client.grpc.Collections.HnswConfigDiff;
|
||||
import io.qdrant.client.grpc.Collections.Modifier;
|
||||
import io.qdrant.client.grpc.Collections.MultiVectorComparator;
|
||||
import io.qdrant.client.grpc.Collections.MultiVectorConfig;
|
||||
import io.qdrant.client.grpc.Collections.SparseVectorConfig;
|
||||
import io.qdrant.client.grpc.Collections.SparseVectorParams;
|
||||
import io.qdrant.client.grpc.Collections.VectorParams;
|
||||
import io.qdrant.client.grpc.Collections.VectorParamsMap;
|
||||
import io.qdrant.client.grpc.Collections.VectorsConfig;
|
||||
import io.qdrant.client.grpc.Points.Document;
|
||||
import io.qdrant.client.grpc.Points.Fusion;
|
||||
import io.qdrant.client.grpc.Points.PointStruct;
|
||||
import io.qdrant.client.grpc.Points.PrefetchQuery;
|
||||
import io.qdrant.client.grpc.Points.Query;
|
||||
import io.qdrant.client.grpc.Points.QueryPoints;
|
||||
import java.io.BufferedReader;
|
||||
import java.io.InputStreamReader;
|
||||
import java.net.URL;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
public class Snippet {
|
||||
|
||||
// @block-start parse-csv
|
||||
static class CsvRow {
|
||||
final String title;
|
||||
final String author;
|
||||
final String description;
|
||||
CsvRow(String title, String author, String description) {
|
||||
this.title = title; this.author = author; this.description = description;
|
||||
}
|
||||
}
|
||||
|
||||
static Stream<CsvRow> parseCSV(String url) throws Exception {
|
||||
Function<String, List<String>> parseCsvLine = line -> {
|
||||
List<String> fields = new ArrayList<>();
|
||||
boolean inQuotes = false;
|
||||
var sb = new StringBuilder();
|
||||
for (char c : line.toCharArray()) {
|
||||
if (c == '"') {
|
||||
inQuotes = !inQuotes;
|
||||
} else if (c == ',' && !inQuotes) {
|
||||
fields.add(sb.toString());
|
||||
sb.setLength(0);
|
||||
} else {
|
||||
sb.append(c);
|
||||
}
|
||||
}
|
||||
fields.add(sb.toString());
|
||||
return fields;
|
||||
};
|
||||
|
||||
var reader = new BufferedReader(new InputStreamReader(new URL(url).openStream()));
|
||||
String headerLine = reader.readLine();
|
||||
List<String> headers = parseCsvLine.apply(headerLine);
|
||||
int titleIdx = headers.indexOf("Title");
|
||||
int authorIdx = headers.indexOf("Author");
|
||||
int descriptionIdx = headers.indexOf("Description");
|
||||
|
||||
return reader.lines()
|
||||
.map(line -> {
|
||||
List<String> fields = parseCsvLine.apply(line);
|
||||
return new CsvRow(fields.get(titleIdx), fields.get(authorIdx), fields.get(descriptionIdx));
|
||||
})
|
||||
.onClose(() -> { try { reader.close(); } catch (Exception ignored) {} });
|
||||
}
|
||||
// @block-end parse-csv
|
||||
|
||||
public static void run() throws Exception {
|
||||
// @hide-start
|
||||
String QDRANT_URL = "xyz-example.eu-central.aws.cloud.qdrant.io";
|
||||
String QDRANT_API_KEY = "<your-api-key>";
|
||||
// @hide-end
|
||||
// @block-start client-connection
|
||||
QdrantClient client =
|
||||
new QdrantClient(
|
||||
QdrantGrpcClient.newBuilder(QDRANT_URL, 6334, true)
|
||||
.withApiKey(QDRANT_API_KEY)
|
||||
.build());
|
||||
// @block-end client-connection
|
||||
|
||||
// @block-start define-models
|
||||
String denseEmbeddingModel = "sentence-transformers/all-MiniLM-L6-v2";
|
||||
String sparseEmbeddingModel = "qdrant/bm25";
|
||||
String lateInteractionEmbeddingModel = "answerdotai/answerai-colbert-small-v1";
|
||||
// @block-end define-models
|
||||
|
||||
// @block-start create-collection
|
||||
String collectionName = "hybrid-search";
|
||||
|
||||
if (client.collectionExistsAsync(collectionName).get()) {
|
||||
client.deleteCollectionAsync(collectionName).get();
|
||||
}
|
||||
|
||||
client.createCollectionAsync(
|
||||
CreateCollection.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.setVectorsConfig(
|
||||
VectorsConfig.newBuilder()
|
||||
.setParamsMap(
|
||||
VectorParamsMap.newBuilder()
|
||||
.putMap(
|
||||
"dense",
|
||||
VectorParams.newBuilder()
|
||||
.setSize(384)
|
||||
.setDistance(Distance.Cosine)
|
||||
.build())
|
||||
.putMap(
|
||||
"multi",
|
||||
VectorParams.newBuilder()
|
||||
.setSize(96)
|
||||
.setDistance(Distance.Cosine)
|
||||
.setMultivectorConfig(
|
||||
MultiVectorConfig.newBuilder()
|
||||
.setComparator(MultiVectorComparator.MaxSim)
|
||||
.build())
|
||||
.setHnswConfig(
|
||||
HnswConfigDiff.newBuilder()
|
||||
.setM(0) // Disable HNSW for reranking
|
||||
.build())
|
||||
.build())
|
||||
.build()))
|
||||
.setSparseVectorsConfig(
|
||||
SparseVectorConfig.newBuilder()
|
||||
.putMap(
|
||||
"sparse",
|
||||
SparseVectorParams.newBuilder()
|
||||
.setModifier(Modifier.Idf)
|
||||
.build())
|
||||
.build())
|
||||
.build()
|
||||
).get();
|
||||
// @block-end create-collection
|
||||
|
||||
// @block-start ingest-data
|
||||
String csvUrl = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/sci-fi-books/top_100_scifi_books_full.csv";
|
||||
|
||||
int batchSize = 25;
|
||||
long idx = 0;
|
||||
List<PointStruct> buffer = new ArrayList<>();
|
||||
|
||||
try (var stream = parseCSV(csvUrl)) {
|
||||
for (var row : (Iterable<CsvRow>) stream::iterator) {
|
||||
String title = row.title;
|
||||
String author = row.author;
|
||||
String description = row.description;
|
||||
|
||||
buffer.add(
|
||||
PointStruct.newBuilder()
|
||||
.setId(io.qdrant.client.PointIdFactory.id(idx++))
|
||||
.setVectors(
|
||||
namedVectors(
|
||||
Map.of(
|
||||
"dense",
|
||||
vector(
|
||||
Document.newBuilder()
|
||||
.setText(description)
|
||||
.setModel(denseEmbeddingModel)
|
||||
.build()),
|
||||
"sparse",
|
||||
vector(
|
||||
Document.newBuilder()
|
||||
.setText(description)
|
||||
.setModel(sparseEmbeddingModel)
|
||||
.build()),
|
||||
"multi",
|
||||
vector(
|
||||
Document.newBuilder()
|
||||
.setText(description)
|
||||
.setModel(lateInteractionEmbeddingModel)
|
||||
.build()))))
|
||||
.putAllPayload(
|
||||
Map.of(
|
||||
"title", value(title),
|
||||
"author", value(author),
|
||||
"description", value(description)))
|
||||
.build());
|
||||
|
||||
if (buffer.size() >= batchSize) {
|
||||
client.upsertAsync(collectionName, buffer).get();
|
||||
buffer.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!buffer.isEmpty()) {
|
||||
client.upsertAsync(collectionName, buffer).get();
|
||||
}
|
||||
// @block-end ingest-data
|
||||
|
||||
// @block-start dense-retrieval
|
||||
String query = "time travel";
|
||||
|
||||
var results = client.queryAsync(
|
||||
QueryPoints.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.setQuery(
|
||||
nearest(
|
||||
Document.newBuilder()
|
||||
.setText(query)
|
||||
.setModel(denseEmbeddingModel)
|
||||
.build()))
|
||||
.setUsing("dense")
|
||||
.setLimit(10)
|
||||
.build()
|
||||
).get();
|
||||
|
||||
for (var result : results) {
|
||||
System.out.println(result);
|
||||
}
|
||||
// @block-end dense-retrieval
|
||||
|
||||
// @block-start sparse-retrieval
|
||||
results = client.queryAsync(
|
||||
QueryPoints.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.setQuery(
|
||||
nearest(
|
||||
Document.newBuilder()
|
||||
.setText(query)
|
||||
.setModel(sparseEmbeddingModel)
|
||||
.build()))
|
||||
.setUsing("sparse")
|
||||
.setLimit(10)
|
||||
.build()
|
||||
).get();
|
||||
|
||||
for (var result : results) {
|
||||
System.out.println(result);
|
||||
}
|
||||
// @block-end sparse-retrieval
|
||||
|
||||
// @block-start hybrid-search
|
||||
results = client.queryAsync(
|
||||
QueryPoints.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.addPrefetch(
|
||||
PrefetchQuery.newBuilder()
|
||||
.setQuery(
|
||||
nearest(
|
||||
Document.newBuilder()
|
||||
.setText(query)
|
||||
.setModel(denseEmbeddingModel)
|
||||
.build()))
|
||||
.setUsing("dense")
|
||||
.setLimit(20)
|
||||
.build())
|
||||
.addPrefetch(
|
||||
PrefetchQuery.newBuilder()
|
||||
.setQuery(
|
||||
nearest(
|
||||
Document.newBuilder()
|
||||
.setText(query)
|
||||
.setModel(sparseEmbeddingModel)
|
||||
.build()))
|
||||
.setUsing("sparse")
|
||||
.setLimit(20)
|
||||
.build())
|
||||
.setQuery(Query.newBuilder().setFusion(Fusion.RRF).build())
|
||||
.setWithPayload(enable(true))
|
||||
.setLimit(10)
|
||||
.build()
|
||||
).get();
|
||||
|
||||
for (var result : results) {
|
||||
System.out.println(result);
|
||||
}
|
||||
// @block-end hybrid-search
|
||||
|
||||
// @block-start rerank
|
||||
results = client.queryAsync(
|
||||
QueryPoints.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.addPrefetch(
|
||||
PrefetchQuery.newBuilder()
|
||||
.setQuery(
|
||||
nearest(
|
||||
Document.newBuilder()
|
||||
.setText(query)
|
||||
.setModel(denseEmbeddingModel)
|
||||
.build()))
|
||||
.setUsing("dense")
|
||||
.setLimit(20)
|
||||
.build())
|
||||
.addPrefetch(
|
||||
PrefetchQuery.newBuilder()
|
||||
.setQuery(
|
||||
nearest(
|
||||
Document.newBuilder()
|
||||
.setText(query)
|
||||
.setModel(sparseEmbeddingModel)
|
||||
.build()))
|
||||
.setUsing("sparse")
|
||||
.setLimit(20)
|
||||
.build())
|
||||
.setQuery(
|
||||
nearest(
|
||||
Document.newBuilder()
|
||||
.setText(query)
|
||||
.setModel(lateInteractionEmbeddingModel)
|
||||
.build()))
|
||||
.setUsing("multi")
|
||||
.setWithPayload(enable(true))
|
||||
.setLimit(10)
|
||||
.build()
|
||||
).get();
|
||||
|
||||
for (var result : results) {
|
||||
System.out.println(result);
|
||||
}
|
||||
// @block-end rerank
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user