krzysztof/landing_page
1
0
Fork 0
mirror of https://github.com/qdrant/landing_page.git synced 2026-10-04 02:18:29 +02:00
Code Issues Packages Projects Releases Wiki Activity

Update Hybrid Search with Reranking tutorial (#2274)

Browse Source 
* Update for Cloud Inference and data ingestion

* Fix link

* Review feedback

* Make code snippets testable

* Add C# code snippets

* Add Go code snippets

* Add Java code snippets

* Add Rust code snippets

* Add TS code snippets

* Move CSV streaming/parsing to separate function
This commit is contained in:
Abdon Pijpelink
2026-04-22 11:03:05 +02:00
committed by GitHub
parent bc0efe70b0
commit 434a451c2b
68 changed files with 3911 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/csharp.cs
+197
View File
@@ -0,0 +1,197 @@
using System.Net.Http;
using Microsoft.VisualBasic.FileIO;
using Qdrant.Client;
using Qdrant.Client.Grpc;
public class Snippet
{
public static async Task Run()
{
// @hide-start
string QDRANT_URL = "xyz-example.eu-central.aws.cloud.qdrant.io";
string QDRANT_API_KEY = "<your-api-key>";
// @hide-end
// @block-start client-connection
var client = new QdrantClient(
host: QDRANT_URL,
https: true,
apiKey: QDRANT_API_KEY
);
// @block-end client-connection
// @block-start define-models
string denseEmbeddingModel = "sentence-transformers/all-MiniLM-L6-v2";
string sparseEmbeddingModel = "qdrant/bm25";
string lateInteractionEmbeddingModel = "answerdotai/answerai-colbert-small-v1";
// @block-end define-models
// @block-start create-collection
string collectionName = "hybrid-search";
if (await client.CollectionExistsAsync(collectionName))
await client.DeleteCollectionAsync(collectionName);
await client.CreateCollectionAsync(
collectionName: collectionName,
vectorsConfig: new VectorParamsMap
{
Map =
{
["dense"] = new VectorParams
{
Size = 384,
Distance = Distance.Cosine,
},
["multi"] = new VectorParams
{
Size = 96,
Distance = Distance.Cosine,
MultivectorConfig = new() { Comparator = MultiVectorComparator.MaxSim },
HnswConfig = new HnswConfigDiff { M = 0 }, // Disable HNSW for reranking
},
}
},
sparseVectorsConfig: new SparseVectorConfig
{
Map =
{
["sparse"] = new SparseVectorParams { Modifier = Modifier.Idf }
}
}
);
// @block-end create-collection
// @block-start parse-csv
async IAsyncEnumerable<(string title, string author, string description)> ParseCsv(string url)
{
using var httpClient = new HttpClient();
using var stream = await httpClient.GetStreamAsync(url);
using var parser = new TextFieldParser(new StreamReader(stream));
parser.TextFieldType = Microsoft.VisualBasic.FileIO.FieldType.Delimited;
parser.SetDelimiters(",");
string[]? headers = parser.ReadFields();
int titleIdx = Array.IndexOf(headers!, "Title");
int authorIdx = Array.IndexOf(headers!, "Author");
int descriptionIdx = Array.IndexOf(headers!, "Description");
while (!parser.EndOfData)
{
var fields = parser.ReadFields()!;
yield return (fields[titleIdx], fields[authorIdx], fields[descriptionIdx]);
}
}
// @block-end parse-csv
// @block-start ingest-data
string csvUrl = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/sci-fi-books/top_100_scifi_books_full.csv";
int batchSize = 25;
ulong idx = 0;
var buffer = new List<PointStruct>();
await foreach (var (title, author, description) in ParseCsv(csvUrl))
{
buffer.Add(new PointStruct
{
Id = idx++,
Vectors = new Dictionary<string, Vector>
{
["dense"] = new Document { Text = description, Model = denseEmbeddingModel },
["sparse"] = new Document { Text = description, Model = sparseEmbeddingModel },
["multi"] = new Document { Text = description, Model = lateInteractionEmbeddingModel },
},
Payload = { ["title"] = title, ["author"] = author, ["description"] = description }
});
if (buffer.Count >= batchSize)
{
await client.UpsertAsync(collectionName: collectionName, points: buffer);
buffer.Clear();
}
}
if (buffer.Count > 0)
await client.UpsertAsync(collectionName: collectionName, points: buffer);
// @block-end ingest-data
// @block-start dense-retrieval
string query = "time travel";
var results = await client.QueryAsync(
collectionName: collectionName,
query: new Document { Text = query, Model = denseEmbeddingModel },
usingVector: "dense",
limit: 10
);
foreach (var result in results)
Console.WriteLine(result);
// @block-end dense-retrieval
// @block-start sparse-retrieval
results = await client.QueryAsync(
collectionName: collectionName,
query: new Document { Text = query, Model = sparseEmbeddingModel },
usingVector: "sparse",
limit: 10
);
foreach (var result in results)
Console.WriteLine(result);
// @block-end sparse-retrieval
// @block-start hybrid-search
results = await client.QueryAsync(
collectionName: collectionName,
prefetch: new List<PrefetchQuery>
{
new()
{
Query = new Document { Text = query, Model = denseEmbeddingModel },
Using = "dense",
Limit = 20,
},
new()
{
Query = new Document { Text = query, Model = sparseEmbeddingModel },
Using = "sparse",
Limit = 20,
},
},
query: Fusion.Rrf,
payloadSelector: true,
limit: 10
);
foreach (var result in results)
Console.WriteLine(result);
// @block-end hybrid-search
// @block-start rerank
results = await client.QueryAsync(
collectionName: collectionName,
prefetch: new List<PrefetchQuery>
{
new()
{
Query = new Document { Text = query, Model = denseEmbeddingModel },
Using = "dense",
Limit = 20,
},
new()
{
Query = new Document { Text = query, Model = sparseEmbeddingModel },
Using = "sparse",
Limit = 20,
},
},
query: new Document { Text = query, Model = lateInteractionEmbeddingModel },
usingVector: "multi",
payloadSelector: true,
limit: 10
);
foreach (var result in results)
Console.WriteLine(result);
// @block-end rerank
}
}
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/client-connection/csharp.md
+7
View File
@@ -0,0 +1,7 @@
```csharp
var client = new QdrantClient(
host: QDRANT_URL,
https: true,
apiKey: QDRANT_API_KEY
);
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/client-connection/go.md
+7
View File
@@ -0,0 +1,7 @@
```go
client, err := qdrant.NewClient(&qdrant.Config{
Host: QDRANT_URL,
APIKey: QDRANT_API_KEY,
UseTLS: true,
})
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/client-connection/java.md
+7
View File
@@ -0,0 +1,7 @@
```java
QdrantClient client =
new QdrantClient(
QdrantGrpcClient.newBuilder(QDRANT_URL, 6334, true)
.withApiKey(QDRANT_API_KEY)
.build());
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/client-connection/python.md
+9
View File
@@ -0,0 +1,9 @@
```python
from qdrant_client import QdrantClient
client = QdrantClient(
url="https://xyz-example.eu-central.aws.cloud.qdrant.io:6333",
api_key="<your-api-key>",
cloud_inference=True,
)
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/client-connection/rust.md
+5
View File
@@ -0,0 +1,5 @@
```rust
let client = Qdrant::from_url(qdrant_url)
.api_key(qdrant_api_key)
.build()?;
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/client-connection/typescript.md
+6
View File
@@ -0,0 +1,6 @@
```typescript
const client = new QdrantClient({
url: QDRANT_URL,
apiKey: QDRANT_API_KEY,
});
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/create-collection/csharp.md
+35
View File
@@ -0,0 +1,35 @@
```csharp
string collectionName = "hybrid-search";
if (await client.CollectionExistsAsync(collectionName))
await client.DeleteCollectionAsync(collectionName);
await client.CreateCollectionAsync(
collectionName: collectionName,
vectorsConfig: new VectorParamsMap
{
Map =
{
["dense"] = new VectorParams
{
Size = 384,
Distance = Distance.Cosine,
},
["multi"] = new VectorParams
{
Size = 96,
Distance = Distance.Cosine,
MultivectorConfig = new() { Comparator = MultiVectorComparator.MaxSim },
HnswConfig = new HnswConfigDiff { M = 0 }, // Disable HNSW for reranking
},
}
},
sparseVectorsConfig: new SparseVectorConfig
{
Map =
{
["sparse"] = new SparseVectorParams { Modifier = Modifier.Idf }
}
}
);
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/create-collection/go.md
+33
View File
@@ -0,0 +1,33 @@
```go
collectionName := "hybrid-search"
exists, err := client.CollectionExists(context.Background(), collectionName)
if exists {
client.DeleteCollection(context.Background(), collectionName)
}
client.CreateCollection(context.Background(), &qdrant.CreateCollection{
CollectionName: collectionName,
VectorsConfig: qdrant.NewVectorsConfigMap(
map[string]*qdrant.VectorParams{
"dense": {
Size: 384,
Distance: qdrant.Distance_Cosine,
},
"multi": {
Size: 96,
Distance: qdrant.Distance_Cosine,
MultivectorConfig: &qdrant.MultiVectorConfig{
Comparator: qdrant.MultiVectorComparator_MaxSim,
},
HnswConfig: &qdrant.HnswConfigDiff{M: qdrant.PtrOf(uint64(0))}, // Disable HNSW for reranking
},
},
),
SparseVectorsConfig: qdrant.NewSparseVectorsConfig(
map[string]*qdrant.SparseVectorParams{
"sparse": {Modifier: qdrant.Modifier_Idf.Enum()},
},
),
})
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/create-collection/java.md
+46
View File
@@ -0,0 +1,46 @@
```java
String collectionName = "hybrid-search";
if (client.collectionExistsAsync(collectionName).get()) {
client.deleteCollectionAsync(collectionName).get();
}
client.createCollectionAsync(
CreateCollection.newBuilder()
.setCollectionName(collectionName)
.setVectorsConfig(
VectorsConfig.newBuilder()
.setParamsMap(
VectorParamsMap.newBuilder()
.putMap(
"dense",
VectorParams.newBuilder()
.setSize(384)
.setDistance(Distance.Cosine)
.build())
.putMap(
"multi",
VectorParams.newBuilder()
.setSize(96)
.setDistance(Distance.Cosine)
.setMultivectorConfig(
MultiVectorConfig.newBuilder()
.setComparator(MultiVectorComparator.MaxSim)
.build())
.setHnswConfig(
HnswConfigDiff.newBuilder()
.setM(0) // Disable HNSW for reranking
.build())
.build())
.build()))
.setSparseVectorsConfig(
SparseVectorConfig.newBuilder()
.putMap(
"sparse",
SparseVectorParams.newBuilder()
.setModifier(Modifier.Idf)
.build())
.build())
.build()
).get();
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/create-collection/python.md
+29
View File
@@ -0,0 +1,29 @@
```python
from qdrant_client.models import Distance, VectorParams, models
collection_name = "hybrid-search"
if client.collection_exists(collection_name=collection_name):
client.delete_collection(collection_name=collection_name)
client.create_collection(
collection_name,
vectors_config={
"dense": models.VectorParams(
size=384,
distance=models.Distance.COSINE,
),
"multi": models.VectorParams(
size=96,
distance=models.Distance.COSINE,
multivector_config=models.MultiVectorConfig(
comparator=models.MultiVectorComparator.MAX_SIM,
),
hnsw_config=models.HnswConfigDiff(m=0) # Disable HNSW for reranking
),
},
sparse_vectors_config={
"sparse": models.SparseVectorParams(modifier=models.Modifier.IDF)
}
)
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/create-collection/rust.md
+33
View File
@@ -0,0 +1,33 @@
```rust
let collection_name = "hybrid-search";
if client.collection_exists(collection_name).await? {
client.delete_collection(collection_name).await?;
}
let mut vectors = VectorsConfigBuilder::default();
vectors.add_named_vector_params(
"dense",
VectorParamsBuilder::new(384, Distance::Cosine),
);
vectors.add_named_vector_params(
"multi",
VectorParamsBuilder::new(96, Distance::Cosine)
.multivector_config(MultiVectorConfigBuilder::new(MultiVectorComparator::MaxSim))
.hnsw_config(HnswConfigDiffBuilder::default().m(0)), // Disable HNSW for reranking
);
let mut sparse = SparseVectorsConfigBuilder::default();
sparse.add_named_vector_params(
"sparse",
SparseVectorParamsBuilder::default().modifier(Modifier::Idf),
);
client
.create_collection(
CreateCollectionBuilder::new(collection_name)
.vectors_config(vectors)
.sparse_vectors_config(sparse),
)
.await?;
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/create-collection/typescript.md
+25
View File
@@ -0,0 +1,25 @@
```typescript
const collectionName = "hybrid-search";
if (await client.collectionExists(collectionName)) {
await client.deleteCollection(collectionName);
}
await client.createCollection(collectionName, {
vectors: {
dense: {
size: 384,
distance: "Cosine",
},
multi: {
size: 96,
distance: "Cosine",
multivector_config: { comparator: "max_sim" },
hnsw_config: { m: 0 }, // Disable HNSW for reranking
},
},
sparse_vectors: {
sparse: { modifier: "idf" },
},
});
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/csharp.md
+171
View File
@@ -0,0 +1,171 @@
```csharp
using System.Net.Http;
using Microsoft.VisualBasic.FileIO;
using Qdrant.Client;
using Qdrant.Client.Grpc;
var client = new QdrantClient(
host: QDRANT_URL,
https: true,
apiKey: QDRANT_API_KEY
);
string denseEmbeddingModel = "sentence-transformers/all-MiniLM-L6-v2";
string sparseEmbeddingModel = "qdrant/bm25";
string lateInteractionEmbeddingModel = "answerdotai/answerai-colbert-small-v1";
string collectionName = "hybrid-search";
if (await client.CollectionExistsAsync(collectionName))
await client.DeleteCollectionAsync(collectionName);
await client.CreateCollectionAsync(
collectionName: collectionName,
vectorsConfig: new VectorParamsMap
{
Map =
{
["dense"] = new VectorParams
{
Size = 384,
Distance = Distance.Cosine,
},
["multi"] = new VectorParams
{
Size = 96,
Distance = Distance.Cosine,
MultivectorConfig = new() { Comparator = MultiVectorComparator.MaxSim },
HnswConfig = new HnswConfigDiff { M = 0 }, // Disable HNSW for reranking
},
}
},
sparseVectorsConfig: new SparseVectorConfig
{
Map =
{
["sparse"] = new SparseVectorParams { Modifier = Modifier.Idf }
}
}
);
async IAsyncEnumerable<(string title, string author, string description)> ParseCsv(string url)
{
using var httpClient = new HttpClient();
using var stream = await httpClient.GetStreamAsync(url);
using var parser = new TextFieldParser(new StreamReader(stream));
parser.TextFieldType = Microsoft.VisualBasic.FileIO.FieldType.Delimited;
parser.SetDelimiters(",");
string[]? headers = parser.ReadFields();
int titleIdx = Array.IndexOf(headers!, "Title");
int authorIdx = Array.IndexOf(headers!, "Author");
int descriptionIdx = Array.IndexOf(headers!, "Description");
while (!parser.EndOfData)
{
var fields = parser.ReadFields()!;
yield return (fields[titleIdx], fields[authorIdx], fields[descriptionIdx]);
}
}
string csvUrl = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/sci-fi-books/top_100_scifi_books_full.csv";
int batchSize = 25;
ulong idx = 0;
var buffer = new List<PointStruct>();
await foreach (var (title, author, description) in ParseCsv(csvUrl))
{
buffer.Add(new PointStruct
{
Id = idx++,
Vectors = new Dictionary<string, Vector>
{
["dense"] = new Document { Text = description, Model = denseEmbeddingModel },
["sparse"] = new Document { Text = description, Model = sparseEmbeddingModel },
["multi"] = new Document { Text = description, Model = lateInteractionEmbeddingModel },
},
Payload = { ["title"] = title, ["author"] = author, ["description"] = description }
});
if (buffer.Count >= batchSize)
{
await client.UpsertAsync(collectionName: collectionName, points: buffer);
buffer.Clear();
}
}
if (buffer.Count > 0)
await client.UpsertAsync(collectionName: collectionName, points: buffer);
string query = "time travel";
var results = await client.QueryAsync(
collectionName: collectionName,
query: new Document { Text = query, Model = denseEmbeddingModel },
usingVector: "dense",
limit: 10
);
foreach (var result in results)
Console.WriteLine(result);
results = await client.QueryAsync(
collectionName: collectionName,
query: new Document { Text = query, Model = sparseEmbeddingModel },
usingVector: "sparse",
limit: 10
);
foreach (var result in results)
Console.WriteLine(result);
results = await client.QueryAsync(
collectionName: collectionName,
prefetch: new List<PrefetchQuery>
{
new()
{
Query = new Document { Text = query, Model = denseEmbeddingModel },
Using = "dense",
Limit = 20,
},
new()
{
Query = new Document { Text = query, Model = sparseEmbeddingModel },
Using = "sparse",
Limit = 20,
},
},
query: Fusion.Rrf,
payloadSelector: true,
limit: 10
);
foreach (var result in results)
Console.WriteLine(result);
results = await client.QueryAsync(
collectionName: collectionName,
prefetch: new List<PrefetchQuery>
{
new()
{
Query = new Document { Text = query, Model = denseEmbeddingModel },
Using = "dense",
Limit = 20,
},
new()
{
Query = new Document { Text = query, Model = sparseEmbeddingModel },
Using = "sparse",
Limit = 20,
},
},
query: new Document { Text = query, Model = lateInteractionEmbeddingModel },
usingVector: "multi",
payloadSelector: true,
limit: 10
);
foreach (var result in results)
Console.WriteLine(result);
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/define-models/csharp.md
+5
View File
@@ -0,0 +1,5 @@
```csharp
string denseEmbeddingModel = "sentence-transformers/all-MiniLM-L6-v2";
string sparseEmbeddingModel = "qdrant/bm25";
string lateInteractionEmbeddingModel = "answerdotai/answerai-colbert-small-v1";
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/define-models/go.md
+5
View File
@@ -0,0 +1,5 @@
```go
denseEmbeddingModel := "sentence-transformers/all-MiniLM-L6-v2"
sparseEmbeddingModel := "qdrant/bm25"
lateInteractionEmbeddingModel := "answerdotai/answerai-colbert-small-v1"
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/define-models/java.md
+5
View File
@@ -0,0 +1,5 @@
```java
String denseEmbeddingModel = "sentence-transformers/all-MiniLM-L6-v2";
String sparseEmbeddingModel = "qdrant/bm25";
String lateInteractionEmbeddingModel = "answerdotai/answerai-colbert-small-v1";
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/define-models/python.md
+5
View File
@@ -0,0 +1,5 @@
```python
dense_embedding_model = "sentence-transformers/all-MiniLM-L6-v2"
sparse_embedding_model = "qdrant/bm25"
late_interaction_embedding_model = "answerdotai/answerai-colbert-small-v1"
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/define-models/rust.md
+5
View File
@@ -0,0 +1,5 @@
```rust
let dense_embedding_model = "sentence-transformers/all-MiniLM-L6-v2";
let sparse_embedding_model = "qdrant/bm25";
let late_interaction_embedding_model = "answerdotai/answerai-colbert-small-v1";
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/define-models/typescript.md
+5
View File
@@ -0,0 +1,5 @@
```typescript
const denseEmbeddingModel = "sentence-transformers/all-MiniLM-L6-v2";
const sparseEmbeddingModel = "qdrant/bm25";
const lateInteractionEmbeddingModel = "answerdotai/answerai-colbert-small-v1";
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/dense-retrieval/csharp.md
+13
View File
@@ -0,0 +1,13 @@
```csharp
string query = "time travel";
var results = await client.QueryAsync(
collectionName: collectionName,
query: new Document { Text = query, Model = denseEmbeddingModel },
usingVector: "dense",
limit: 10
);
foreach (var result in results)
Console.WriteLine(result);
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/dense-retrieval/go.md
+17
View File
@@ -0,0 +1,17 @@
```go
query := "time travel"
results, err := client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: collectionName,
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: query,
Model: denseEmbeddingModel,
}),
Using: qdrant.PtrOf("dense"),
Limit: qdrant.PtrOf(uint64(10)),
})
for _, result := range results {
fmt.Println(result)
}
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/dense-retrieval/java.md
+21
View File
@@ -0,0 +1,21 @@
```java
String query = "time travel";
var results = client.queryAsync(
QueryPoints.newBuilder()
.setCollectionName(collectionName)
.setQuery(
nearest(
Document.newBuilder()
.setText(query)
.setModel(denseEmbeddingModel)
.build()))
.setUsing("dense")
.setLimit(10)
.build()
).get();
for (var result : results) {
System.out.println(result);
}
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/dense-retrieval/python.md
+14
View File
@@ -0,0 +1,14 @@
```python
import pprint
query = "time travel"
results = client.query_points(
collection_name,
query=models.Document(text=query, model=dense_embedding_model),
using="dense",
limit=10,
)
pprint.pp(results.points)
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/dense-retrieval/rust.md
+16
View File
@@ -0,0 +1,16 @@
```rust
let query = "time travel";
let results = client
.query(
QueryPointsBuilder::new(collection_name)
.query(Query::new_nearest(Document::new(query, dense_embedding_model)))
.using("dense")
.limit(10),
)
.await?;
for result in results.result {
println!("{:?}", result);
}
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/dense-retrieval/typescript.md
+11
View File
@@ -0,0 +1,11 @@
```typescript
const query = "time travel";
const denseResults = await client.query(collectionName, {
query: { text: query, model: denseEmbeddingModel },
using: "dense",
limit: 10,
});
console.log(denseResults.points);
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/go.md
+231
View File
@@ -0,0 +1,231 @@
```go
import (
"context"
"encoding/csv"
"fmt"
"io"
"net/http"
"github.com/qdrant/go-client/qdrant"
)
type CSVRow struct {
Title string
Author string
Description string
}
func parseCSV(url string, fn func(CSVRow)) error {
resp, err := http.Get(url)
if err != nil {
return err
}
defer resp.Body.Close()
csvReader := csv.NewReader(resp.Body)
headers, err := csvReader.Read()
if err != nil {
return err
}
titleIdx, authorIdx, descriptionIdx := -1, -1, -1
for i, h := range headers {
switch h {
case "Title":
titleIdx = i
case "Author":
authorIdx = i
case "Description":
descriptionIdx = i
}
}
for {
row, err := csvReader.Read()
if err == io.EOF {
break
}
if err != nil {
return err
}
fn(CSVRow{Title: row[titleIdx], Author: row[authorIdx], Description: row[descriptionIdx]})
}
return nil
}
client, err := qdrant.NewClient(&qdrant.Config{
Host: QDRANT_URL,
APIKey: QDRANT_API_KEY,
UseTLS: true,
})
denseEmbeddingModel := "sentence-transformers/all-MiniLM-L6-v2"
sparseEmbeddingModel := "qdrant/bm25"
lateInteractionEmbeddingModel := "answerdotai/answerai-colbert-small-v1"
collectionName := "hybrid-search"
exists, err := client.CollectionExists(context.Background(), collectionName)
if exists {
client.DeleteCollection(context.Background(), collectionName)
}
client.CreateCollection(context.Background(), &qdrant.CreateCollection{
CollectionName: collectionName,
VectorsConfig: qdrant.NewVectorsConfigMap(
map[string]*qdrant.VectorParams{
"dense": {
Size: 384,
Distance: qdrant.Distance_Cosine,
},
"multi": {
Size: 96,
Distance: qdrant.Distance_Cosine,
MultivectorConfig: &qdrant.MultiVectorConfig{
Comparator: qdrant.MultiVectorComparator_MaxSim,
},
HnswConfig: &qdrant.HnswConfigDiff{M: qdrant.PtrOf(uint64(0))}, // Disable HNSW for reranking
},
},
),
SparseVectorsConfig: qdrant.NewSparseVectorsConfig(
map[string]*qdrant.SparseVectorParams{
"sparse": {Modifier: qdrant.Modifier_Idf.Enum()},
},
),
})
csvUrl := "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/sci-fi-books/top_100_scifi_books_full.csv"
batchSize := 25
var idx uint64
var buffer []*qdrant.PointStruct
err = parseCSV(csvUrl, func(row CSVRow) {
title := row.Title
author := row.Author
description := row.Description
buffer = append(buffer, &qdrant.PointStruct{
Id: qdrant.NewIDNum(idx),
Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{
"dense": qdrant.NewVectorDocument(&qdrant.Document{Text: description, Model: denseEmbeddingModel}),
"sparse": qdrant.NewVectorDocument(&qdrant.Document{Text: description, Model: sparseEmbeddingModel}),
"multi": qdrant.NewVectorDocument(&qdrant.Document{Text: description, Model: lateInteractionEmbeddingModel}),
}),
Payload: qdrant.NewValueMap(map[string]any{
"title": title,
"author": author,
"description": description,
}),
})
idx++
if len(buffer) >= batchSize {
client.Upsert(context.Background(), &qdrant.UpsertPoints{
CollectionName: collectionName,
Points: buffer,
})
buffer = nil
}
})
if len(buffer) > 0 {
client.Upsert(context.Background(), &qdrant.UpsertPoints{
CollectionName: collectionName,
Points: buffer,
})
}
query := "time travel"
results, err := client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: collectionName,
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: query,
Model: denseEmbeddingModel,
}),
Using: qdrant.PtrOf("dense"),
Limit: qdrant.PtrOf(uint64(10)),
})
for _, result := range results {
fmt.Println(result)
}
results, err = client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: collectionName,
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: query,
Model: sparseEmbeddingModel,
}),
Using: qdrant.PtrOf("sparse"),
Limit: qdrant.PtrOf(uint64(10)),
})
for _, result := range results {
fmt.Println(result)
}
results, err = client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: collectionName,
Prefetch: []*qdrant.PrefetchQuery{
{
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: query,
Model: denseEmbeddingModel,
}),
Using: qdrant.PtrOf("dense"),
Limit: qdrant.PtrOf(uint64(20)),
},
{
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: query,
Model: sparseEmbeddingModel,
}),
Using: qdrant.PtrOf("sparse"),
Limit: qdrant.PtrOf(uint64(20)),
},
},
Query: qdrant.NewQueryFusion(qdrant.Fusion_RRF),
WithPayload: qdrant.NewWithPayload(true),
Limit: qdrant.PtrOf(uint64(10)),
})
for _, result := range results {
fmt.Println(result)
}
results, err = client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: collectionName,
Prefetch: []*qdrant.PrefetchQuery{
{
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: query,
Model: denseEmbeddingModel,
}),
Using: qdrant.PtrOf("dense"),
Limit: qdrant.PtrOf(uint64(20)),
},
{
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: query,
Model: sparseEmbeddingModel,
}),
Using: qdrant.PtrOf("sparse"),
Limit: qdrant.PtrOf(uint64(20)),
},
},
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: query,
Model: lateInteractionEmbeddingModel,
}),
Using: qdrant.PtrOf("multi"),
WithPayload: qdrant.NewWithPayload(true),
Limit: qdrant.PtrOf(uint64(10)),
})
for _, result := range results {
fmt.Println(result)
}
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/hybrid-search/csharp.md
+26
View File
@@ -0,0 +1,26 @@
```csharp
results = await client.QueryAsync(
collectionName: collectionName,
prefetch: new List<PrefetchQuery>
{
new()
{
Query = new Document { Text = query, Model = denseEmbeddingModel },
Using = "dense",
Limit = 20,
},
new()
{
Query = new Document { Text = query, Model = sparseEmbeddingModel },
Using = "sparse",
Limit = 20,
},
},
query: Fusion.Rrf,
payloadSelector: true,
limit: 10
);
foreach (var result in results)
Console.WriteLine(result);
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/hybrid-search/go.md
+30
View File
@@ -0,0 +1,30 @@
```go
results, err = client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: collectionName,
Prefetch: []*qdrant.PrefetchQuery{
{
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: query,
Model: denseEmbeddingModel,
}),
Using: qdrant.PtrOf("dense"),
Limit: qdrant.PtrOf(uint64(20)),
},
{
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: query,
Model: sparseEmbeddingModel,
}),
Using: qdrant.PtrOf("sparse"),
Limit: qdrant.PtrOf(uint64(20)),
},
},
Query: qdrant.NewQueryFusion(qdrant.Fusion_RRF),
WithPayload: qdrant.NewWithPayload(true),
Limit: qdrant.PtrOf(uint64(10)),
})
for _, result := range results {
fmt.Println(result)
}
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/hybrid-search/java.md
+36
View File
@@ -0,0 +1,36 @@
```java
results = client.queryAsync(
QueryPoints.newBuilder()
.setCollectionName(collectionName)
.addPrefetch(
PrefetchQuery.newBuilder()
.setQuery(
nearest(
Document.newBuilder()
.setText(query)
.setModel(denseEmbeddingModel)
.build()))
.setUsing("dense")
.setLimit(20)
.build())
.addPrefetch(
PrefetchQuery.newBuilder()
.setQuery(
nearest(
Document.newBuilder()
.setText(query)
.setModel(sparseEmbeddingModel)
.build()))
.setUsing("sparse")
.setLimit(20)
.build())
.setQuery(Query.newBuilder().setFusion(Fusion.RRF).build())
.setWithPayload(enable(true))
.setLimit(10)
.build()
).get();
for (var result : results) {
System.out.println(result);
}
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/hybrid-search/python.md
+24
View File
@@ -0,0 +1,24 @@
```python
prefetch = [
models.Prefetch(
query=models.Document(text=query, model=dense_embedding_model),
using="dense",
limit=20,
),
models.Prefetch(
query=models.Document(text=query, model=sparse_embedding_model),
using="sparse",
limit=20,
),
]
results = client.query_points(
collection_name,
prefetch=prefetch,
query=models.FusionQuery(fusion=models.Fusion.RRF),
with_payload=True,
limit=10,
)
pprint.pp(results.points)
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/hybrid-search/rust.md
+26
View File
@@ -0,0 +1,26 @@
```rust
let results = client
.query(
QueryPointsBuilder::new(collection_name)
.add_prefetch(
PrefetchQueryBuilder::default()
.query(Query::new_nearest(Document::new(query, dense_embedding_model)))
.using("dense")
.limit(20u64),
)
.add_prefetch(
PrefetchQueryBuilder::default()
.query(Query::new_nearest(Document::new(query, sparse_embedding_model)))
.using("sparse")
.limit(20u64),
)
.query(Query::new_fusion(Fusion::Rrf))
.with_payload(true)
.limit(10),
)
.await?;
for result in results.result {
println!("{:?}", result);
}
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/hybrid-search/typescript.md
+21
View File
@@ -0,0 +1,21 @@
```typescript
const hybridResults = await client.query(collectionName, {
prefetch: [
{
query: { text: query, model: denseEmbeddingModel },
using: "dense",
limit: 20,
},
{
query: { text: query, model: sparseEmbeddingModel },
using: "sparse",
limit: 20,
},
],
query: { fusion: "rrf" },
with_payload: true,
limit: 10,
});
console.log(hybridResults.points);
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/ingest-data/csharp.md
+31
View File
@@ -0,0 +1,31 @@
```csharp
string csvUrl = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/sci-fi-books/top_100_scifi_books_full.csv";
int batchSize = 25;
ulong idx = 0;
var buffer = new List<PointStruct>();
await foreach (var (title, author, description) in ParseCsv(csvUrl))
{
buffer.Add(new PointStruct
{
Id = idx++,
Vectors = new Dictionary<string, Vector>
{
["dense"] = new Document { Text = description, Model = denseEmbeddingModel },
["sparse"] = new Document { Text = description, Model = sparseEmbeddingModel },
["multi"] = new Document { Text = description, Model = lateInteractionEmbeddingModel },
},
Payload = { ["title"] = title, ["author"] = author, ["description"] = description }
});
if (buffer.Count >= batchSize)
{
await client.UpsertAsync(collectionName: collectionName, points: buffer);
buffer.Clear();
}
}
if (buffer.Count > 0)
await client.UpsertAsync(collectionName: collectionName, points: buffer);
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/ingest-data/go.md
+43
View File
@@ -0,0 +1,43 @@
```go
csvUrl := "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/sci-fi-books/top_100_scifi_books_full.csv"
batchSize := 25
var idx uint64
var buffer []*qdrant.PointStruct
err = parseCSV(csvUrl, func(row CSVRow) {
title := row.Title
author := row.Author
description := row.Description
buffer = append(buffer, &qdrant.PointStruct{
Id: qdrant.NewIDNum(idx),
Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{
"dense": qdrant.NewVectorDocument(&qdrant.Document{Text: description, Model: denseEmbeddingModel}),
"sparse": qdrant.NewVectorDocument(&qdrant.Document{Text: description, Model: sparseEmbeddingModel}),
"multi": qdrant.NewVectorDocument(&qdrant.Document{Text: description, Model: lateInteractionEmbeddingModel}),
}),
Payload: qdrant.NewValueMap(map[string]any{
"title": title,
"author": author,
"description": description,
}),
})
idx++
if len(buffer) >= batchSize {
client.Upsert(context.Background(), &qdrant.UpsertPoints{
CollectionName: collectionName,
Points: buffer,
})
buffer = nil
}
})
if len(buffer) > 0 {
client.Upsert(context.Background(), &qdrant.UpsertPoints{
CollectionName: collectionName,
Points: buffer,
})
}
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/ingest-data/java.md
+55
View File
@@ -0,0 +1,55 @@
```java
String csvUrl = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/sci-fi-books/top_100_scifi_books_full.csv";
int batchSize = 25;
long idx = 0;
List<PointStruct> buffer = new ArrayList<>();
try (var stream = parseCSV(csvUrl)) {
for (var row : (Iterable<CsvRow>) stream::iterator) {
String title = row.title;
String author = row.author;
String description = row.description;
buffer.add(
PointStruct.newBuilder()
.setId(io.qdrant.client.PointIdFactory.id(idx++))
.setVectors(
namedVectors(
Map.of(
"dense",
vector(
Document.newBuilder()
.setText(description)
.setModel(denseEmbeddingModel)
.build()),
"sparse",
vector(
Document.newBuilder()
.setText(description)
.setModel(sparseEmbeddingModel)
.build()),
"multi",
vector(
Document.newBuilder()
.setText(description)
.setModel(lateInteractionEmbeddingModel)
.build()))))
.putAllPayload(
Map.of(
"title", value(title),
"author", value(author),
"description", value(description)))
.build());
if (buffer.size() >= batchSize) {
client.upsertAsync(collectionName, buffer).get();
buffer.clear();
}
}
}
if (!buffer.isEmpty()) {
client.upsertAsync(collectionName, buffer).get();
}
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/ingest-data/python.md
+23
View File
@@ -0,0 +1,23 @@
```python
from qdrant_client.models import Document, PointStruct
csv_url = 'https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/sci-fi-books/top_100_scifi_books_full.csv'
points = (
PointStruct(
id=idx,
vector={
"dense": Document(text=row['Description'], model=dense_embedding_model),
"sparse": Document(text=row['Description'], model=sparse_embedding_model),
"multi": Document(text=row['Description'], model=late_interaction_embedding_model),
},
payload={"title": row['Title'], "author": row['Author'], "description": row['Description']}
)
for idx, row in enumerate(parse_csv(csv_url))
)
client.upload_points(
collection_name=collection_name,
points=points,
batch_size=25
)
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/ingest-data/rust.md
+45
View File
@@ -0,0 +1,45 @@
```rust
let csv_url = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/sci-fi-books/top_100_scifi_books_full.csv";
let batch_size = 25;
let mut idx: u64 = 0;
let mut buffer: Vec<PointStruct> = Vec::new();
for row in parse_csv(csv_url)? {
let row = row?;
let title = row.title;
let author = row.author;
let description = row.description;
let vectors = NamedVectors::default()
.add_vector("dense", Document::new(&description, dense_embedding_model))
.add_vector("sparse", Document::new(&description, sparse_embedding_model))
.add_vector("multi", Document::new(&description, late_interaction_embedding_model));
buffer.push(PointStruct::new(
idx,
vectors,
[
("title", title.into()),
("author", author.into()),
("description", description.into()),
],
));
idx += 1;
if buffer.len() >= batch_size {
client
.upsert_points(UpsertPointsBuilder::new(
collection_name,
std::mem::take(&mut buffer),
))
.await?;
}
}
if !buffer.is_empty() {
client
.upsert_points(UpsertPointsBuilder::new(collection_name, buffer))
.await?;
}
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/ingest-data/typescript.md
+28
View File
@@ -0,0 +1,28 @@
```typescript
const csvUrl = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/sci-fi-books/top_100_scifi_books_full.csv";
const batchSize = 25;
let idx = 0;
let buffer: Schemas["PointStruct"][] = [];
for await (const { title, author, description } of parseCSV(csvUrl)) {
buffer.push({
id: idx++,
vector: {
dense: { text: description, model: denseEmbeddingModel },
sparse: { text: description, model: sparseEmbeddingModel },
multi: { text: description, model: lateInteractionEmbeddingModel },
},
payload: { title, author, description },
});
if (buffer.length >= batchSize) {
await client.upsert(collectionName, { points: buffer });
buffer = [];
}
}
if (buffer.length > 0) {
await client.upsert(collectionName, { points: buffer });
}
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/java.md
+301
View File
@@ -0,0 +1,301 @@
```java
import static io.qdrant.client.QueryFactory.nearest;
import static io.qdrant.client.ValueFactory.value;
import static io.qdrant.client.VectorFactory.vector;
import static io.qdrant.client.VectorsFactory.namedVectors;
import static io.qdrant.client.WithPayloadSelectorFactory.enable;
import io.qdrant.client.QdrantClient;
import io.qdrant.client.QdrantGrpcClient;
import io.qdrant.client.grpc.Collections.CreateCollection;
import io.qdrant.client.grpc.Collections.Distance;
import io.qdrant.client.grpc.Collections.HnswConfigDiff;
import io.qdrant.client.grpc.Collections.Modifier;
import io.qdrant.client.grpc.Collections.MultiVectorComparator;
import io.qdrant.client.grpc.Collections.MultiVectorConfig;
import io.qdrant.client.grpc.Collections.SparseVectorConfig;
import io.qdrant.client.grpc.Collections.SparseVectorParams;
import io.qdrant.client.grpc.Collections.VectorParams;
import io.qdrant.client.grpc.Collections.VectorParamsMap;
import io.qdrant.client.grpc.Collections.VectorsConfig;
import io.qdrant.client.grpc.Points.Document;
import io.qdrant.client.grpc.Points.Fusion;
import io.qdrant.client.grpc.Points.PointStruct;
import io.qdrant.client.grpc.Points.PrefetchQuery;
import io.qdrant.client.grpc.Points.Query;
import io.qdrant.client.grpc.Points.QueryPoints;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Stream;
static class CsvRow {
final String title;
final String author;
final String description;
CsvRow(String title, String author, String description) {
this.title = title; this.author = author; this.description = description;
}
}
static Stream<CsvRow> parseCSV(String url) throws Exception {
Function<String, List<String>> parseCsvLine = line -> {
List<String> fields = new ArrayList<>();
boolean inQuotes = false;
var sb = new StringBuilder();
for (char c : line.toCharArray()) {
if (c == '"') {
inQuotes = !inQuotes;
} else if (c == ',' && !inQuotes) {
fields.add(sb.toString());
sb.setLength(0);
} else {
sb.append(c);
}
}
fields.add(sb.toString());
return fields;
};
var reader = new BufferedReader(new InputStreamReader(new URL(url).openStream()));
String headerLine = reader.readLine();
List<String> headers = parseCsvLine.apply(headerLine);
int titleIdx = headers.indexOf("Title");
int authorIdx = headers.indexOf("Author");
int descriptionIdx = headers.indexOf("Description");
return reader.lines()
.map(line -> {
List<String> fields = parseCsvLine.apply(line);
return new CsvRow(fields.get(titleIdx), fields.get(authorIdx), fields.get(descriptionIdx));
})
.onClose(() -> { try { reader.close(); } catch (Exception ignored) {} });
}
QdrantClient client =
new QdrantClient(
QdrantGrpcClient.newBuilder(QDRANT_URL, 6334, true)
.withApiKey(QDRANT_API_KEY)
.build());
String denseEmbeddingModel = "sentence-transformers/all-MiniLM-L6-v2";
String sparseEmbeddingModel = "qdrant/bm25";
String lateInteractionEmbeddingModel = "answerdotai/answerai-colbert-small-v1";
String collectionName = "hybrid-search";
if (client.collectionExistsAsync(collectionName).get()) {
client.deleteCollectionAsync(collectionName).get();
}
client.createCollectionAsync(
CreateCollection.newBuilder()
.setCollectionName(collectionName)
.setVectorsConfig(
VectorsConfig.newBuilder()
.setParamsMap(
VectorParamsMap.newBuilder()
.putMap(
"dense",
VectorParams.newBuilder()
.setSize(384)
.setDistance(Distance.Cosine)
.build())
.putMap(
"multi",
VectorParams.newBuilder()
.setSize(96)
.setDistance(Distance.Cosine)
.setMultivectorConfig(
MultiVectorConfig.newBuilder()
.setComparator(MultiVectorComparator.MaxSim)
.build())
.setHnswConfig(
HnswConfigDiff.newBuilder()
.setM(0) // Disable HNSW for reranking
.build())
.build())
.build()))
.setSparseVectorsConfig(
SparseVectorConfig.newBuilder()
.putMap(
"sparse",
SparseVectorParams.newBuilder()
.setModifier(Modifier.Idf)
.build())
.build())
.build()
).get();
String csvUrl = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/sci-fi-books/top_100_scifi_books_full.csv";
int batchSize = 25;
long idx = 0;
List<PointStruct> buffer = new ArrayList<>();
try (var stream = parseCSV(csvUrl)) {
for (var row : (Iterable<CsvRow>) stream::iterator) {
String title = row.title;
String author = row.author;
String description = row.description;
buffer.add(
PointStruct.newBuilder()
.setId(io.qdrant.client.PointIdFactory.id(idx++))
.setVectors(
namedVectors(
Map.of(
"dense",
vector(
Document.newBuilder()
.setText(description)
.setModel(denseEmbeddingModel)
.build()),
"sparse",
vector(
Document.newBuilder()
.setText(description)
.setModel(sparseEmbeddingModel)
.build()),
"multi",
vector(
Document.newBuilder()
.setText(description)
.setModel(lateInteractionEmbeddingModel)
.build()))))
.putAllPayload(
Map.of(
"title", value(title),
"author", value(author),
"description", value(description)))
.build());
if (buffer.size() >= batchSize) {
client.upsertAsync(collectionName, buffer).get();
buffer.clear();
}
}
}
if (!buffer.isEmpty()) {
client.upsertAsync(collectionName, buffer).get();
}
String query = "time travel";
var results = client.queryAsync(
QueryPoints.newBuilder()
.setCollectionName(collectionName)
.setQuery(
nearest(
Document.newBuilder()
.setText(query)
.setModel(denseEmbeddingModel)
.build()))
.setUsing("dense")
.setLimit(10)
.build()
).get();
for (var result : results) {
System.out.println(result);
}
results = client.queryAsync(
QueryPoints.newBuilder()
.setCollectionName(collectionName)
.setQuery(
nearest(
Document.newBuilder()
.setText(query)
.setModel(sparseEmbeddingModel)
.build()))
.setUsing("sparse")
.setLimit(10)
.build()
).get();
for (var result : results) {
System.out.println(result);
}
results = client.queryAsync(
QueryPoints.newBuilder()
.setCollectionName(collectionName)
.addPrefetch(
PrefetchQuery.newBuilder()
.setQuery(
nearest(
Document.newBuilder()
.setText(query)
.setModel(denseEmbeddingModel)
.build()))
.setUsing("dense")
.setLimit(20)
.build())
.addPrefetch(
PrefetchQuery.newBuilder()
.setQuery(
nearest(
Document.newBuilder()
.setText(query)
.setModel(sparseEmbeddingModel)
.build()))
.setUsing("sparse")
.setLimit(20)
.build())
.setQuery(Query.newBuilder().setFusion(Fusion.RRF).build())
.setWithPayload(enable(true))
.setLimit(10)
.build()
).get();
for (var result : results) {
System.out.println(result);
}
results = client.queryAsync(
QueryPoints.newBuilder()
.setCollectionName(collectionName)
.addPrefetch(
PrefetchQuery.newBuilder()
.setQuery(
nearest(
Document.newBuilder()
.setText(query)
.setModel(denseEmbeddingModel)
.build()))
.setUsing("dense")
.setLimit(20)
.build())
.addPrefetch(
PrefetchQuery.newBuilder()
.setQuery(
nearest(
Document.newBuilder()
.setText(query)
.setModel(sparseEmbeddingModel)
.build()))
.setUsing("sparse")
.setLimit(20)
.build())
.setQuery(
nearest(
Document.newBuilder()
.setText(query)
.setModel(lateInteractionEmbeddingModel)
.build()))
.setUsing("multi")
.setWithPayload(enable(true))
.setLimit(10)
.build()
).get();
for (var result : results) {
System.out.println(result);
}
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/parse-csv/csharp.md
+19
View File
@@ -0,0 +1,19 @@
```csharp
async IAsyncEnumerable<(string title, string author, string description)> ParseCsv(string url)
{
using var httpClient = new HttpClient();
using var stream = await httpClient.GetStreamAsync(url);
using var parser = new TextFieldParser(new StreamReader(stream));
parser.TextFieldType = Microsoft.VisualBasic.FileIO.FieldType.Delimited;
parser.SetDelimiters(",");
string[]? headers = parser.ReadFields();
int titleIdx = Array.IndexOf(headers!, "Title");
int authorIdx = Array.IndexOf(headers!, "Author");
int descriptionIdx = Array.IndexOf(headers!, "Description");
while (!parser.EndOfData)
{
var fields = parser.ReadFields()!;
yield return (fields[titleIdx], fields[authorIdx], fields[descriptionIdx]);
}
}
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/parse-csv/go.md
+45
View File
@@ -0,0 +1,45 @@
```go
type CSVRow struct {
Title string
Author string
Description string
}
func parseCSV(url string, fn func(CSVRow)) error {
resp, err := http.Get(url)
if err != nil {
return err
}
defer resp.Body.Close()
csvReader := csv.NewReader(resp.Body)
headers, err := csvReader.Read()
if err != nil {
return err
}
titleIdx, authorIdx, descriptionIdx := -1, -1, -1
for i, h := range headers {
switch h {
case "Title":
titleIdx = i
case "Author":
authorIdx = i
case "Description":
descriptionIdx = i
}
}
for {
row, err := csvReader.Read()
if err == io.EOF {
break
}
if err != nil {
return err
}
fn(CSVRow{Title: row[titleIdx], Author: row[authorIdx], Description: row[descriptionIdx]})
}
return nil
}
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/parse-csv/java.md
+44
View File
@@ -0,0 +1,44 @@
```java
static class CsvRow {
final String title;
final String author;
final String description;
CsvRow(String title, String author, String description) {
this.title = title; this.author = author; this.description = description;
}
}
static Stream<CsvRow> parseCSV(String url) throws Exception {
Function<String, List<String>> parseCsvLine = line -> {
List<String> fields = new ArrayList<>();
boolean inQuotes = false;
var sb = new StringBuilder();
for (char c : line.toCharArray()) {
if (c == '"') {
inQuotes = !inQuotes;
} else if (c == ',' && !inQuotes) {
fields.add(sb.toString());
sb.setLength(0);
} else {
sb.append(c);
}
}
fields.add(sb.toString());
return fields;
};
var reader = new BufferedReader(new InputStreamReader(new URL(url).openStream()));
String headerLine = reader.readLine();
List<String> headers = parseCsvLine.apply(headerLine);
int titleIdx = headers.indexOf("Title");
int authorIdx = headers.indexOf("Author");
int descriptionIdx = headers.indexOf("Description");
return reader.lines()
.map(line -> {
List<String> fields = parseCsvLine.apply(line);
return new CsvRow(fields.get(titleIdx), fields.get(authorIdx), fields.get(descriptionIdx));
})
.onClose(() -> { try { reader.close(); } catch (Exception ignored) {} });
}
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/parse-csv/python.md
+9
View File
@@ -0,0 +1,9 @@
```python
import csv
import urllib.request
def parse_csv(url):
with urllib.request.urlopen(url) as response:
reader = csv.DictReader(line.decode('utf-8') for line in response)
yield from reader
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/parse-csv/rust.md
+25
View File
@@ -0,0 +1,25 @@
```rust
struct CsvRow {
title: String,
author: String,
description: String,
}
fn parse_csv(url: &str) -> anyhow::Result<impl Iterator<Item = anyhow::Result<CsvRow>>> {
let reader = ureq::get(url).call()?.into_body().into_reader();
let mut rdr = csv::Reader::from_reader(reader);
let headers = rdr.headers()?.clone();
let title_idx = headers.iter().position(|h| h == "Title").unwrap();
let author_idx = headers.iter().position(|h| h == "Author").unwrap();
let description_idx = headers.iter().position(|h| h == "Description").unwrap();
let iter = rdr.into_records().map(move |result| {
let record = result?;
Ok(CsvRow {
title: record[title_idx].to_string(),
author: record[author_idx].to_string(),
description: record[description_idx].to_string(),
})
});
Ok(iter)
}
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/parse-csv/typescript.md
+58
View File
@@ -0,0 +1,58 @@
```typescript
function parseCsvLine(line: string): string[] {
const fields: string[] = [];
let i = 0;
while (i < line.length) {
if (line[i] === '"') {
i++;
let field = "";
while (i < line.length) {
if (line[i] === '"' && line[i + 1] === '"') { field += '"'; i += 2; }
else if (line[i] === '"') { i++; break; }
else { field += line[i++]; }
}
fields.push(field);
if (line[i] === ",") i++;
} else {
const start = i;
while (i < line.length && line[i] !== ",") i++;
fields.push(line.slice(start, i));
if (i < line.length) i++;
}
}
return fields;
}
async function* parseCSV(url: string): AsyncGenerator<{ title: string; author: string; description: string }> {
const response = await fetch(url);
const reader = response.body!.getReader();
const decoder = new TextDecoder();
let remainder = "";
let headers: string[] | null = null;
let titleIdx = -1;
let authorIdx = -1;
let descriptionIdx = -1;
while (true) {
const { done, value } = await reader.read();
const chunk = done ? "" : decoder.decode(value, { stream: true });
const lines = (remainder + chunk).split("\n");
remainder = done ? "" : lines.pop()!;
for (const line of lines) {
if (!line.trim()) continue;
if (headers === null) {
headers = parseCsvLine(line);
titleIdx = headers.indexOf("Title");
authorIdx = headers.indexOf("Author");
descriptionIdx = headers.indexOf("Description");
continue;
}
const fields = parseCsvLine(line);
yield { title: fields[titleIdx], author: fields[authorIdx], description: fields[descriptionIdx] };
}
if (done) break;
}
}
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/python.md
+140
View File
@@ -0,0 +1,140 @@
```python
from qdrant_client import QdrantClient
client = QdrantClient(
url="https://xyz-example.eu-central.aws.cloud.qdrant.io:6333",
api_key="<your-api-key>",
cloud_inference=True,
)
dense_embedding_model = "sentence-transformers/all-MiniLM-L6-v2"
sparse_embedding_model = "qdrant/bm25"
late_interaction_embedding_model = "answerdotai/answerai-colbert-small-v1"
from qdrant_client.models import Distance, VectorParams, models
collection_name = "hybrid-search"
if client.collection_exists(collection_name=collection_name):
client.delete_collection(collection_name=collection_name)
client.create_collection(
collection_name,
vectors_config={
"dense": models.VectorParams(
size=384,
distance=models.Distance.COSINE,
),
"multi": models.VectorParams(
size=96,
distance=models.Distance.COSINE,
multivector_config=models.MultiVectorConfig(
comparator=models.MultiVectorComparator.MAX_SIM,
),
hnsw_config=models.HnswConfigDiff(m=0) # Disable HNSW for reranking
),
},
sparse_vectors_config={
"sparse": models.SparseVectorParams(modifier=models.Modifier.IDF)
}
)
import csv
import urllib.request
def parse_csv(url):
with urllib.request.urlopen(url) as response:
reader = csv.DictReader(line.decode('utf-8') for line in response)
yield from reader
from qdrant_client.models import Document, PointStruct
csv_url = 'https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/sci-fi-books/top_100_scifi_books_full.csv'
points = (
PointStruct(
id=idx,
vector={
"dense": Document(text=row['Description'], model=dense_embedding_model),
"sparse": Document(text=row['Description'], model=sparse_embedding_model),
"multi": Document(text=row['Description'], model=late_interaction_embedding_model),
},
payload={"title": row['Title'], "author": row['Author'], "description": row['Description']}
)
for idx, row in enumerate(parse_csv(csv_url))
)
client.upload_points(
collection_name=collection_name,
points=points,
batch_size=25
)
import pprint
query = "time travel"
results = client.query_points(
collection_name,
query=models.Document(text=query, model=dense_embedding_model),
using="dense",
limit=10,
)
pprint.pp(results.points)
results = client.query_points(
collection_name,
query=models.Document(text=query, model=sparse_embedding_model),
using="sparse",
limit=10,
)
pprint.pp(results.points)
prefetch = [
models.Prefetch(
query=models.Document(text=query, model=dense_embedding_model),
using="dense",
limit=20,
),
models.Prefetch(
query=models.Document(text=query, model=sparse_embedding_model),
using="sparse",
limit=20,
),
]
results = client.query_points(
collection_name,
prefetch=prefetch,
query=models.FusionQuery(fusion=models.Fusion.RRF),
with_payload=True,
limit=10,
)
pprint.pp(results.points)
prefetch = [
models.Prefetch(
query=models.Document(text=query, model=dense_embedding_model),
using="dense",
limit=20,
),
models.Prefetch(
query=models.Document(text=query, model=sparse_embedding_model),
using="sparse",
limit=20,
),
]
results = client.query_points(
collection_name,
prefetch=prefetch,
query=models.Document(text=query, model=late_interaction_embedding_model),
using="multi",
with_payload=True,
limit=10,
)
pprint.pp(results.points)
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/rerank/csharp.md
+27
View File
@@ -0,0 +1,27 @@
```csharp
results = await client.QueryAsync(
collectionName: collectionName,
prefetch: new List<PrefetchQuery>
{
new()
{
Query = new Document { Text = query, Model = denseEmbeddingModel },
Using = "dense",
Limit = 20,
},
new()
{
Query = new Document { Text = query, Model = sparseEmbeddingModel },
Using = "sparse",
Limit = 20,
},
},
query: new Document { Text = query, Model = lateInteractionEmbeddingModel },
usingVector: "multi",
payloadSelector: true,
limit: 10
);
foreach (var result in results)
Console.WriteLine(result);
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/rerank/go.md
+34
View File
@@ -0,0 +1,34 @@
```go
results, err = client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: collectionName,
Prefetch: []*qdrant.PrefetchQuery{
{
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: query,
Model: denseEmbeddingModel,
}),
Using: qdrant.PtrOf("dense"),
Limit: qdrant.PtrOf(uint64(20)),
},
{
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: query,
Model: sparseEmbeddingModel,
}),
Using: qdrant.PtrOf("sparse"),
Limit: qdrant.PtrOf(uint64(20)),
},
},
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: query,
Model: lateInteractionEmbeddingModel,
}),
Using: qdrant.PtrOf("multi"),
WithPayload: qdrant.NewWithPayload(true),
Limit: qdrant.PtrOf(uint64(10)),
})
for _, result := range results {
fmt.Println(result)
}
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/rerank/java.md
+42
View File
@@ -0,0 +1,42 @@
```java
results = client.queryAsync(
QueryPoints.newBuilder()
.setCollectionName(collectionName)
.addPrefetch(
PrefetchQuery.newBuilder()
.setQuery(
nearest(
Document.newBuilder()
.setText(query)
.setModel(denseEmbeddingModel)
.build()))
.setUsing("dense")
.setLimit(20)
.build())
.addPrefetch(
PrefetchQuery.newBuilder()
.setQuery(
nearest(
Document.newBuilder()
.setText(query)
.setModel(sparseEmbeddingModel)
.build()))
.setUsing("sparse")
.setLimit(20)
.build())
.setQuery(
nearest(
Document.newBuilder()
.setText(query)
.setModel(lateInteractionEmbeddingModel)
.build()))
.setUsing("multi")
.setWithPayload(enable(true))
.setLimit(10)
.build()
).get();
for (var result : results) {
System.out.println(result);
}
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/rerank/python.md
+25
View File
@@ -0,0 +1,25 @@
```python
prefetch = [
models.Prefetch(
query=models.Document(text=query, model=dense_embedding_model),
using="dense",
limit=20,
),
models.Prefetch(
query=models.Document(text=query, model=sparse_embedding_model),
using="sparse",
limit=20,
),
]
results = client.query_points(
collection_name,
prefetch=prefetch,
query=models.Document(text=query, model=late_interaction_embedding_model),
using="multi",
with_payload=True,
limit=10,
)
pprint.pp(results.points)
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/rerank/rust.md
+27
View File
@@ -0,0 +1,27 @@
```rust
let results = client
.query(
QueryPointsBuilder::new(collection_name)
.add_prefetch(
PrefetchQueryBuilder::default()
.query(Query::new_nearest(Document::new(query, dense_embedding_model)))
.using("dense")
.limit(20u64),
)
.add_prefetch(
PrefetchQueryBuilder::default()
.query(Query::new_nearest(Document::new(query, sparse_embedding_model)))
.using("sparse")
.limit(20u64),
)
.query(Query::new_nearest(Document::new(query, late_interaction_embedding_model)))
.using("multi")
.with_payload(true)
.limit(10),
)
.await?;
for result in results.result {
println!("{:?}", result);
}
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/rerank/typescript.md
+22
View File
@@ -0,0 +1,22 @@
```typescript
const rerankedResults = await client.query(collectionName, {
prefetch: [
{
query: { text: query, model: denseEmbeddingModel },
using: "dense",
limit: 20,
},
{
query: { text: query, model: sparseEmbeddingModel },
using: "sparse",
limit: 20,
},
],
query: { text: query, model: lateInteractionEmbeddingModel },
using: "multi",
with_payload: true,
limit: 10,
});
console.log(rerankedResults.points);
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/rust.md
+196
View File
@@ -0,0 +1,196 @@
```rust
use qdrant_client::Qdrant;
use qdrant_client::qdrant::{
CreateCollectionBuilder, Distance, Document, Fusion, HnswConfigDiffBuilder,
Modifier, MultiVectorComparator, MultiVectorConfigBuilder, NamedVectors, PointStruct,
PrefetchQueryBuilder, Query, QueryPointsBuilder, SparseVectorParamsBuilder,
SparseVectorsConfigBuilder, UpsertPointsBuilder, VectorParamsBuilder, VectorsConfigBuilder,
};
let client = Qdrant::from_url(qdrant_url)
.api_key(qdrant_api_key)
.build()?;
let dense_embedding_model = "sentence-transformers/all-MiniLM-L6-v2";
let sparse_embedding_model = "qdrant/bm25";
let late_interaction_embedding_model = "answerdotai/answerai-colbert-small-v1";
let collection_name = "hybrid-search";
if client.collection_exists(collection_name).await? {
client.delete_collection(collection_name).await?;
}
let mut vectors = VectorsConfigBuilder::default();
vectors.add_named_vector_params(
"dense",
VectorParamsBuilder::new(384, Distance::Cosine),
);
vectors.add_named_vector_params(
"multi",
VectorParamsBuilder::new(96, Distance::Cosine)
.multivector_config(MultiVectorConfigBuilder::new(MultiVectorComparator::MaxSim))
.hnsw_config(HnswConfigDiffBuilder::default().m(0)), // Disable HNSW for reranking
);
let mut sparse = SparseVectorsConfigBuilder::default();
sparse.add_named_vector_params(
"sparse",
SparseVectorParamsBuilder::default().modifier(Modifier::Idf),
);
client
.create_collection(
CreateCollectionBuilder::new(collection_name)
.vectors_config(vectors)
.sparse_vectors_config(sparse),
)
.await?;
struct CsvRow {
title: String,
author: String,
description: String,
}
fn parse_csv(url: &str) -> anyhow::Result<impl Iterator<Item = anyhow::Result<CsvRow>>> {
let reader = ureq::get(url).call()?.into_body().into_reader();
let mut rdr = csv::Reader::from_reader(reader);
let headers = rdr.headers()?.clone();
let title_idx = headers.iter().position(|h| h == "Title").unwrap();
let author_idx = headers.iter().position(|h| h == "Author").unwrap();
let description_idx = headers.iter().position(|h| h == "Description").unwrap();
let iter = rdr.into_records().map(move |result| {
let record = result?;
Ok(CsvRow {
title: record[title_idx].to_string(),
author: record[author_idx].to_string(),
description: record[description_idx].to_string(),
})
});
Ok(iter)
}
let csv_url = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/sci-fi-books/top_100_scifi_books_full.csv";
let batch_size = 25;
let mut idx: u64 = 0;
let mut buffer: Vec<PointStruct> = Vec::new();
for row in parse_csv(csv_url)? {
let row = row?;
let title = row.title;
let author = row.author;
let description = row.description;
let vectors = NamedVectors::default()
.add_vector("dense", Document::new(&description, dense_embedding_model))
.add_vector("sparse", Document::new(&description, sparse_embedding_model))
.add_vector("multi", Document::new(&description, late_interaction_embedding_model));
buffer.push(PointStruct::new(
idx,
vectors,
[
("title", title.into()),
("author", author.into()),
("description", description.into()),
],
));
idx += 1;
if buffer.len() >= batch_size {
client
.upsert_points(UpsertPointsBuilder::new(
collection_name,
std::mem::take(&mut buffer),
))
.await?;
}
}
if !buffer.is_empty() {
client
.upsert_points(UpsertPointsBuilder::new(collection_name, buffer))
.await?;
}
let query = "time travel";
let results = client
.query(
QueryPointsBuilder::new(collection_name)
.query(Query::new_nearest(Document::new(query, dense_embedding_model)))
.using("dense")
.limit(10),
)
.await?;
for result in results.result {
println!("{:?}", result);
}
let results = client
.query(
QueryPointsBuilder::new(collection_name)
.query(Query::new_nearest(Document::new(query, sparse_embedding_model)))
.using("sparse")
.limit(10),
)
.await?;
for result in results.result {
println!("{:?}", result);
}
let results = client
.query(
QueryPointsBuilder::new(collection_name)
.add_prefetch(
PrefetchQueryBuilder::default()
.query(Query::new_nearest(Document::new(query, dense_embedding_model)))
.using("dense")
.limit(20u64),
)
.add_prefetch(
PrefetchQueryBuilder::default()
.query(Query::new_nearest(Document::new(query, sparse_embedding_model)))
.using("sparse")
.limit(20u64),
)
.query(Query::new_fusion(Fusion::Rrf))
.with_payload(true)
.limit(10),
)
.await?;
for result in results.result {
println!("{:?}", result);
}
let results = client
.query(
QueryPointsBuilder::new(collection_name)
.add_prefetch(
PrefetchQueryBuilder::default()
.query(Query::new_nearest(Document::new(query, dense_embedding_model)))
.using("dense")
.limit(20u64),
)
.add_prefetch(
PrefetchQueryBuilder::default()
.query(Query::new_nearest(Document::new(query, sparse_embedding_model)))
.using("sparse")
.limit(20u64),
)
.query(Query::new_nearest(Document::new(query, late_interaction_embedding_model)))
.using("multi")
.with_payload(true)
.limit(10),
)
.await?;
for result in results.result {
println!("{:?}", result);
}
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/sparse-retrieval/csharp.md
+11
View File
@@ -0,0 +1,11 @@
```csharp
results = await client.QueryAsync(
collectionName: collectionName,
query: new Document { Text = query, Model = sparseEmbeddingModel },
usingVector: "sparse",
limit: 10
);
foreach (var result in results)
Console.WriteLine(result);
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/sparse-retrieval/go.md
+15
View File
@@ -0,0 +1,15 @@
```go
results, err = client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: collectionName,
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: query,
Model: sparseEmbeddingModel,
}),
Using: qdrant.PtrOf("sparse"),
Limit: qdrant.PtrOf(uint64(10)),
})
for _, result := range results {
fmt.Println(result)
}
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/sparse-retrieval/java.md
+19
View File
@@ -0,0 +1,19 @@
```java
results = client.queryAsync(
QueryPoints.newBuilder()
.setCollectionName(collectionName)
.setQuery(
nearest(
Document.newBuilder()
.setText(query)
.setModel(sparseEmbeddingModel)
.build()))
.setUsing("sparse")
.setLimit(10)
.build()
).get();
for (var result : results) {
System.out.println(result);
}
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/sparse-retrieval/python.md
+10
View File
@@ -0,0 +1,10 @@
```python
results = client.query_points(
collection_name,
query=models.Document(text=query, model=sparse_embedding_model),
using="sparse",
limit=10,
)
pprint.pp(results.points)
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/sparse-retrieval/rust.md
+14
View File
@@ -0,0 +1,14 @@
```rust
let results = client
.query(
QueryPointsBuilder::new(collection_name)
.query(Query::new_nearest(Document::new(query, sparse_embedding_model)))
.using("sparse")
.limit(10),
)
.await?;
for result in results.result {
println!("{:?}", result);
}
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/sparse-retrieval/typescript.md
+9
View File
@@ -0,0 +1,9 @@
```typescript
const sparseResults = await client.query(collectionName, {
query: { text: query, model: sparseEmbeddingModel },
using: "sparse",
limit: 10,
});
console.log(sparseResults.points);
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/generated/typescript.md
+179
View File
@@ -0,0 +1,179 @@
```typescript
import { QdrantClient, Schemas } from "@qdrant/js-client-rest";
const client = new QdrantClient({
url: QDRANT_URL,
apiKey: QDRANT_API_KEY,
});
const denseEmbeddingModel = "sentence-transformers/all-MiniLM-L6-v2";
const sparseEmbeddingModel = "qdrant/bm25";
const lateInteractionEmbeddingModel = "answerdotai/answerai-colbert-small-v1";
const collectionName = "hybrid-search";
if (await client.collectionExists(collectionName)) {
await client.deleteCollection(collectionName);
}
await client.createCollection(collectionName, {
vectors: {
dense: {
size: 384,
distance: "Cosine",
},
multi: {
size: 96,
distance: "Cosine",
multivector_config: { comparator: "max_sim" },
hnsw_config: { m: 0 }, // Disable HNSW for reranking
},
},
sparse_vectors: {
sparse: { modifier: "idf" },
},
});
function parseCsvLine(line: string): string[] {
const fields: string[] = [];
let i = 0;
while (i < line.length) {
if (line[i] === '"') {
i++;
let field = "";
while (i < line.length) {
if (line[i] === '"' && line[i + 1] === '"') { field += '"'; i += 2; }
else if (line[i] === '"') { i++; break; }
else { field += line[i++]; }
}
fields.push(field);
if (line[i] === ",") i++;
} else {
const start = i;
while (i < line.length && line[i] !== ",") i++;
fields.push(line.slice(start, i));
if (i < line.length) i++;
}
}
return fields;
}
async function* parseCSV(url: string): AsyncGenerator<{ title: string; author: string; description: string }> {
const response = await fetch(url);
const reader = response.body!.getReader();
const decoder = new TextDecoder();
let remainder = "";
let headers: string[] | null = null;
let titleIdx = -1;
let authorIdx = -1;
let descriptionIdx = -1;
while (true) {
const { done, value } = await reader.read();
const chunk = done ? "" : decoder.decode(value, { stream: true });
const lines = (remainder + chunk).split("\n");
remainder = done ? "" : lines.pop()!;
for (const line of lines) {
if (!line.trim()) continue;
if (headers === null) {
headers = parseCsvLine(line);
titleIdx = headers.indexOf("Title");
authorIdx = headers.indexOf("Author");
descriptionIdx = headers.indexOf("Description");
continue;
}
const fields = parseCsvLine(line);
yield { title: fields[titleIdx], author: fields[authorIdx], description: fields[descriptionIdx] };
}
if (done) break;
}
}
const csvUrl = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/sci-fi-books/top_100_scifi_books_full.csv";
const batchSize = 25;
let idx = 0;
let buffer: Schemas["PointStruct"][] = [];
for await (const { title, author, description } of parseCSV(csvUrl)) {
buffer.push({
id: idx++,
vector: {
dense: { text: description, model: denseEmbeddingModel },
sparse: { text: description, model: sparseEmbeddingModel },
multi: { text: description, model: lateInteractionEmbeddingModel },
},
payload: { title, author, description },
});
if (buffer.length >= batchSize) {
await client.upsert(collectionName, { points: buffer });
buffer = [];
}
}
if (buffer.length > 0) {
await client.upsert(collectionName, { points: buffer });
}
const query = "time travel";
const denseResults = await client.query(collectionName, {
query: { text: query, model: denseEmbeddingModel },
using: "dense",
limit: 10,
});
console.log(denseResults.points);
const sparseResults = await client.query(collectionName, {
query: { text: query, model: sparseEmbeddingModel },
using: "sparse",
limit: 10,
});
console.log(sparseResults.points);
const hybridResults = await client.query(collectionName, {
prefetch: [
{
query: { text: query, model: denseEmbeddingModel },
using: "dense",
limit: 20,
},
{
query: { text: query, model: sparseEmbeddingModel },
using: "sparse",
limit: 20,
},
],
query: { fusion: "rrf" },
with_payload: true,
limit: 10,
});
console.log(hybridResults.points);
const rerankedResults = await client.query(collectionName, {
prefetch: [
{
query: { text: query, model: denseEmbeddingModel },
using: "dense",
limit: 20,
},
{
query: { text: query, model: sparseEmbeddingModel },
using: "sparse",
limit: 20,
},
],
query: { text: query, model: lateInteractionEmbeddingModel },
using: "multi",
with_payload: true,
limit: 10,
});
console.log(rerankedResults.points);
```
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/go.go
+287
View File
@@ -0,0 +1,287 @@
package snippet
import (
"context"
"encoding/csv"
"fmt"
"io"
"net/http"
"github.com/qdrant/go-client/qdrant"
)
// @block-start parse-csv
type CSVRow struct {
Title string
Author string
Description string
}
func parseCSV(url string, fn func(CSVRow)) error {
resp, err := http.Get(url)
if err != nil {
return err
}
defer resp.Body.Close()
csvReader := csv.NewReader(resp.Body)
headers, err := csvReader.Read()
if err != nil {
return err
}
titleIdx, authorIdx, descriptionIdx := -1, -1, -1
for i, h := range headers {
switch h {
case "Title":
titleIdx = i
case "Author":
authorIdx = i
case "Description":
descriptionIdx = i
}
}
for {
row, err := csvReader.Read()
if err == io.EOF {
break
}
if err != nil {
return err
}
fn(CSVRow{Title: row[titleIdx], Author: row[authorIdx], Description: row[descriptionIdx]})
}
return nil
}
// @block-end parse-csv
func Main() {
// @hide-start
QDRANT_URL := "xyz-example.eu-central.aws.cloud.qdrant.io"
QDRANT_API_KEY := "<your-api-key>"
// @hide-end
// @block-start client-connection
client, err := qdrant.NewClient(&qdrant.Config{
Host: QDRANT_URL,
APIKey: QDRANT_API_KEY,
UseTLS: true,
})
// @block-end client-connection
// @hide-start
if err != nil {
panic(err)
}
// @hide-end
// @block-start define-models
denseEmbeddingModel := "sentence-transformers/all-MiniLM-L6-v2"
sparseEmbeddingModel := "qdrant/bm25"
lateInteractionEmbeddingModel := "answerdotai/answerai-colbert-small-v1"
// @block-end define-models
// @block-start create-collection
collectionName := "hybrid-search"
exists, err := client.CollectionExists(context.Background(), collectionName)
if err != nil { panic(err) } // @hide
if exists {
client.DeleteCollection(context.Background(), collectionName)
}
client.CreateCollection(context.Background(), &qdrant.CreateCollection{
CollectionName: collectionName,
VectorsConfig: qdrant.NewVectorsConfigMap(
map[string]*qdrant.VectorParams{
"dense": {
Size: 384,
Distance: qdrant.Distance_Cosine,
},
"multi": {
Size: 96,
Distance: qdrant.Distance_Cosine,
MultivectorConfig: &qdrant.MultiVectorConfig{
Comparator: qdrant.MultiVectorComparator_MaxSim,
},
HnswConfig: &qdrant.HnswConfigDiff{M: qdrant.PtrOf(uint64(0))}, // Disable HNSW for reranking
},
},
),
SparseVectorsConfig: qdrant.NewSparseVectorsConfig(
map[string]*qdrant.SparseVectorParams{
"sparse": {Modifier: qdrant.Modifier_Idf.Enum()},
},
),
})
// @block-end create-collection
// @block-start ingest-data
csvUrl := "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/sci-fi-books/top_100_scifi_books_full.csv"
batchSize := 25
var idx uint64
var buffer []*qdrant.PointStruct
err = parseCSV(csvUrl, func(row CSVRow) {
title := row.Title
author := row.Author
description := row.Description
buffer = append(buffer, &qdrant.PointStruct{
Id: qdrant.NewIDNum(idx),
Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{
"dense": qdrant.NewVectorDocument(&qdrant.Document{Text: description, Model: denseEmbeddingModel}),
"sparse": qdrant.NewVectorDocument(&qdrant.Document{Text: description, Model: sparseEmbeddingModel}),
"multi": qdrant.NewVectorDocument(&qdrant.Document{Text: description, Model: lateInteractionEmbeddingModel}),
}),
Payload: qdrant.NewValueMap(map[string]any{
"title": title,
"author": author,
"description": description,
}),
})
idx++
if len(buffer) >= batchSize {
client.Upsert(context.Background(), &qdrant.UpsertPoints{
CollectionName: collectionName,
Points: buffer,
})
buffer = nil
}
})
if err != nil { panic(err) } // @hide
if len(buffer) > 0 {
client.Upsert(context.Background(), &qdrant.UpsertPoints{
CollectionName: collectionName,
Points: buffer,
})
}
// @block-end ingest-data
// @block-start dense-retrieval
query := "time travel"
results, err := client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: collectionName,
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: query,
Model: denseEmbeddingModel,
}),
Using: qdrant.PtrOf("dense"),
Limit: qdrant.PtrOf(uint64(10)),
})
// @hide-start
if err != nil {
panic(err)
}
// @hide-end
for _, result := range results {
fmt.Println(result)
}
// @block-end dense-retrieval
// @block-start sparse-retrieval
results, err = client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: collectionName,
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: query,
Model: sparseEmbeddingModel,
}),
Using: qdrant.PtrOf("sparse"),
Limit: qdrant.PtrOf(uint64(10)),
})
// @hide-start
if err != nil {
panic(err)
}
// @hide-end
for _, result := range results {
fmt.Println(result)
}
// @block-end sparse-retrieval
// @block-start hybrid-search
results, err = client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: collectionName,
Prefetch: []*qdrant.PrefetchQuery{
{
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: query,
Model: denseEmbeddingModel,
}),
Using: qdrant.PtrOf("dense"),
Limit: qdrant.PtrOf(uint64(20)),
},
{
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: query,
Model: sparseEmbeddingModel,
}),
Using: qdrant.PtrOf("sparse"),
Limit: qdrant.PtrOf(uint64(20)),
},
},
Query: qdrant.NewQueryFusion(qdrant.Fusion_RRF),
WithPayload: qdrant.NewWithPayload(true),
Limit: qdrant.PtrOf(uint64(10)),
})
// @hide-start
if err != nil {
panic(err)
}
// @hide-end
for _, result := range results {
fmt.Println(result)
}
// @block-end hybrid-search
// @block-start rerank
results, err = client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: collectionName,
Prefetch: []*qdrant.PrefetchQuery{
{
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: query,
Model: denseEmbeddingModel,
}),
Using: qdrant.PtrOf("dense"),
Limit: qdrant.PtrOf(uint64(20)),
},
{
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: query,
Model: sparseEmbeddingModel,
}),
Using: qdrant.PtrOf("sparse"),
Limit: qdrant.PtrOf(uint64(20)),
},
},
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: query,
Model: lateInteractionEmbeddingModel,
}),
Using: qdrant.PtrOf("multi"),
WithPayload: qdrant.NewWithPayload(true),
Limit: qdrant.PtrOf(uint64(10)),
})
// @hide-start
if err != nil {
panic(err)
}
// @hide-end
for _, result := range results {
fmt.Println(result)
}
// @block-end rerank
}
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/java.java
+328
View File
@@ -0,0 +1,328 @@
package com.example.snippets_amalgamation;
import static io.qdrant.client.QueryFactory.nearest;
import static io.qdrant.client.ValueFactory.value;
import static io.qdrant.client.VectorFactory.vector;
import static io.qdrant.client.VectorsFactory.namedVectors;
import static io.qdrant.client.WithPayloadSelectorFactory.enable;
import io.qdrant.client.QdrantClient;
import io.qdrant.client.QdrantGrpcClient;
import io.qdrant.client.grpc.Collections.CreateCollection;
import io.qdrant.client.grpc.Collections.Distance;
import io.qdrant.client.grpc.Collections.HnswConfigDiff;
import io.qdrant.client.grpc.Collections.Modifier;
import io.qdrant.client.grpc.Collections.MultiVectorComparator;
import io.qdrant.client.grpc.Collections.MultiVectorConfig;
import io.qdrant.client.grpc.Collections.SparseVectorConfig;
import io.qdrant.client.grpc.Collections.SparseVectorParams;
import io.qdrant.client.grpc.Collections.VectorParams;
import io.qdrant.client.grpc.Collections.VectorParamsMap;
import io.qdrant.client.grpc.Collections.VectorsConfig;
import io.qdrant.client.grpc.Points.Document;
import io.qdrant.client.grpc.Points.Fusion;
import io.qdrant.client.grpc.Points.PointStruct;
import io.qdrant.client.grpc.Points.PrefetchQuery;
import io.qdrant.client.grpc.Points.Query;
import io.qdrant.client.grpc.Points.QueryPoints;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Stream;
public class Snippet {
// @block-start parse-csv
static class CsvRow {
final String title;
final String author;
final String description;
CsvRow(String title, String author, String description) {
this.title = title; this.author = author; this.description = description;
}
}
static Stream<CsvRow> parseCSV(String url) throws Exception {
Function<String, List<String>> parseCsvLine = line -> {
List<String> fields = new ArrayList<>();
boolean inQuotes = false;
var sb = new StringBuilder();
for (char c : line.toCharArray()) {
if (c == '"') {
inQuotes = !inQuotes;
} else if (c == ',' && !inQuotes) {
fields.add(sb.toString());
sb.setLength(0);
} else {
sb.append(c);
}
}
fields.add(sb.toString());
return fields;
};
var reader = new BufferedReader(new InputStreamReader(new URL(url).openStream()));
String headerLine = reader.readLine();
List<String> headers = parseCsvLine.apply(headerLine);
int titleIdx = headers.indexOf("Title");
int authorIdx = headers.indexOf("Author");
int descriptionIdx = headers.indexOf("Description");
return reader.lines()
.map(line -> {
List<String> fields = parseCsvLine.apply(line);
return new CsvRow(fields.get(titleIdx), fields.get(authorIdx), fields.get(descriptionIdx));
})
.onClose(() -> { try { reader.close(); } catch (Exception ignored) {} });
}
// @block-end parse-csv
public static void run() throws Exception {
// @hide-start
String QDRANT_URL = "xyz-example.eu-central.aws.cloud.qdrant.io";
String QDRANT_API_KEY = "<your-api-key>";
// @hide-end
// @block-start client-connection
QdrantClient client =
new QdrantClient(
QdrantGrpcClient.newBuilder(QDRANT_URL, 6334, true)
.withApiKey(QDRANT_API_KEY)
.build());
// @block-end client-connection
// @block-start define-models
String denseEmbeddingModel = "sentence-transformers/all-MiniLM-L6-v2";
String sparseEmbeddingModel = "qdrant/bm25";
String lateInteractionEmbeddingModel = "answerdotai/answerai-colbert-small-v1";
// @block-end define-models
// @block-start create-collection
String collectionName = "hybrid-search";
if (client.collectionExistsAsync(collectionName).get()) {
client.deleteCollectionAsync(collectionName).get();
}
client.createCollectionAsync(
CreateCollection.newBuilder()
.setCollectionName(collectionName)
.setVectorsConfig(
VectorsConfig.newBuilder()
.setParamsMap(
VectorParamsMap.newBuilder()
.putMap(
"dense",
VectorParams.newBuilder()
.setSize(384)
.setDistance(Distance.Cosine)
.build())
.putMap(
"multi",
VectorParams.newBuilder()
.setSize(96)
.setDistance(Distance.Cosine)
.setMultivectorConfig(
MultiVectorConfig.newBuilder()
.setComparator(MultiVectorComparator.MaxSim)
.build())
.setHnswConfig(
HnswConfigDiff.newBuilder()
.setM(0) // Disable HNSW for reranking
.build())
.build())
.build()))
.setSparseVectorsConfig(
SparseVectorConfig.newBuilder()
.putMap(
"sparse",
SparseVectorParams.newBuilder()
.setModifier(Modifier.Idf)
.build())
.build())
.build()
).get();
// @block-end create-collection
// @block-start ingest-data
String csvUrl = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/sci-fi-books/top_100_scifi_books_full.csv";
int batchSize = 25;
long idx = 0;
List<PointStruct> buffer = new ArrayList<>();
try (var stream = parseCSV(csvUrl)) {
for (var row : (Iterable<CsvRow>) stream::iterator) {
String title = row.title;
String author = row.author;
String description = row.description;
buffer.add(
PointStruct.newBuilder()
.setId(io.qdrant.client.PointIdFactory.id(idx++))
.setVectors(
namedVectors(
Map.of(
"dense",
vector(
Document.newBuilder()
.setText(description)
.setModel(denseEmbeddingModel)
.build()),
"sparse",
vector(
Document.newBuilder()
.setText(description)
.setModel(sparseEmbeddingModel)
.build()),
"multi",
vector(
Document.newBuilder()
.setText(description)
.setModel(lateInteractionEmbeddingModel)
.build()))))
.putAllPayload(
Map.of(
"title", value(title),
"author", value(author),
"description", value(description)))
.build());
if (buffer.size() >= batchSize) {
client.upsertAsync(collectionName, buffer).get();
buffer.clear();
}
}
}
if (!buffer.isEmpty()) {
client.upsertAsync(collectionName, buffer).get();
}
// @block-end ingest-data
// @block-start dense-retrieval
String query = "time travel";
var results = client.queryAsync(
QueryPoints.newBuilder()
.setCollectionName(collectionName)
.setQuery(
nearest(
Document.newBuilder()
.setText(query)
.setModel(denseEmbeddingModel)
.build()))
.setUsing("dense")
.setLimit(10)
.build()
).get();
for (var result : results) {
System.out.println(result);
}
// @block-end dense-retrieval
// @block-start sparse-retrieval
results = client.queryAsync(
QueryPoints.newBuilder()
.setCollectionName(collectionName)
.setQuery(
nearest(
Document.newBuilder()
.setText(query)
.setModel(sparseEmbeddingModel)
.build()))
.setUsing("sparse")
.setLimit(10)
.build()
).get();
for (var result : results) {
System.out.println(result);
}
// @block-end sparse-retrieval
// @block-start hybrid-search
results = client.queryAsync(
QueryPoints.newBuilder()
.setCollectionName(collectionName)
.addPrefetch(
PrefetchQuery.newBuilder()
.setQuery(
nearest(
Document.newBuilder()
.setText(query)
.setModel(denseEmbeddingModel)
.build()))
.setUsing("dense")
.setLimit(20)
.build())
.addPrefetch(
PrefetchQuery.newBuilder()
.setQuery(
nearest(
Document.newBuilder()
.setText(query)
.setModel(sparseEmbeddingModel)
.build()))
.setUsing("sparse")
.setLimit(20)
.build())
.setQuery(Query.newBuilder().setFusion(Fusion.RRF).build())
.setWithPayload(enable(true))
.setLimit(10)
.build()
).get();
for (var result : results) {
System.out.println(result);
}
// @block-end hybrid-search
// @block-start rerank
results = client.queryAsync(
QueryPoints.newBuilder()
.setCollectionName(collectionName)
.addPrefetch(
PrefetchQuery.newBuilder()
.setQuery(
nearest(
Document.newBuilder()
.setText(query)
.setModel(denseEmbeddingModel)
.build()))
.setUsing("dense")
.setLimit(20)
.build())
.addPrefetch(
PrefetchQuery.newBuilder()
.setQuery(
nearest(
Document.newBuilder()
.setText(query)
.setModel(sparseEmbeddingModel)
.build()))
.setUsing("sparse")
.setLimit(20)
.build())
.setQuery(
nearest(
Document.newBuilder()
.setText(query)
.setModel(lateInteractionEmbeddingModel)
.build()))
.setUsing("multi")
.setWithPayload(enable(true))
.setLimit(10)
.build()
).get();
for (var result : results) {
System.out.println(result);
}
// @block-end rerank
}
}
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/python.py
+159
View File
@@ -0,0 +1,159 @@
# @hide-start
# mypy: disable-error-code="arg-type"
# @hide-end
# @block-start client-connection
from qdrant_client import QdrantClient
client = QdrantClient(
url="https://xyz-example.eu-central.aws.cloud.qdrant.io:6333",
api_key="<your-api-key>",
cloud_inference=True,
)
# @block-end client-connection
# @block-start define-models
dense_embedding_model = "sentence-transformers/all-MiniLM-L6-v2"
sparse_embedding_model = "qdrant/bm25"
late_interaction_embedding_model = "answerdotai/answerai-colbert-small-v1"
# @block-end define-models
# @block-start create-collection
from qdrant_client.models import Distance, VectorParams, models
collection_name = "hybrid-search"
if client.collection_exists(collection_name=collection_name):
client.delete_collection(collection_name=collection_name)
client.create_collection(
collection_name,
vectors_config={
"dense": models.VectorParams(
size=384,
distance=models.Distance.COSINE,
),
"multi": models.VectorParams(
size=96,
distance=models.Distance.COSINE,
multivector_config=models.MultiVectorConfig(
comparator=models.MultiVectorComparator.MAX_SIM,
),
hnsw_config=models.HnswConfigDiff(m=0) # Disable HNSW for reranking
),
},
sparse_vectors_config={
"sparse": models.SparseVectorParams(modifier=models.Modifier.IDF)
}
)
# @block-end create-collection
# @block-start parse-csv
import csv
import urllib.request
def parse_csv(url):
with urllib.request.urlopen(url) as response:
reader = csv.DictReader(line.decode('utf-8') for line in response)
yield from reader
# @block-end parse-csv
# @block-start ingest-data
from qdrant_client.models import Document, PointStruct
csv_url = 'https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/sci-fi-books/top_100_scifi_books_full.csv'
points = (
PointStruct(
id=idx,
vector={
"dense": Document(text=row['Description'], model=dense_embedding_model),
"sparse": Document(text=row['Description'], model=sparse_embedding_model),
"multi": Document(text=row['Description'], model=late_interaction_embedding_model),
},
payload={"title": row['Title'], "author": row['Author'], "description": row['Description']}
)
for idx, row in enumerate(parse_csv(csv_url))
)
client.upload_points(
collection_name=collection_name,
points=points,
batch_size=25
)
# @block-end ingest-data
# @block-start dense-retrieval
import pprint
query = "time travel"
results = client.query_points(
collection_name,
query=models.Document(text=query, model=dense_embedding_model),
using="dense",
limit=10,
)
pprint.pp(results.points)
# @block-end dense-retrieval
# @block-start sparse-retrieval
results = client.query_points(
collection_name,
query=models.Document(text=query, model=sparse_embedding_model),
using="sparse",
limit=10,
)
pprint.pp(results.points)
# @block-end sparse-retrieval
# @block-start hybrid-search
prefetch = [
models.Prefetch(
query=models.Document(text=query, model=dense_embedding_model),
using="dense",
limit=20,
),
models.Prefetch(
query=models.Document(text=query, model=sparse_embedding_model),
using="sparse",
limit=20,
),
]
results = client.query_points(
collection_name,
prefetch=prefetch,
query=models.FusionQuery(fusion=models.Fusion.RRF),
with_payload=True,
limit=10,
)
pprint.pp(results.points)
# @block-end hybrid-search
# @block-start rerank
prefetch = [
models.Prefetch(
query=models.Document(text=query, model=dense_embedding_model),
using="dense",
limit=20,
),
models.Prefetch(
query=models.Document(text=query, model=sparse_embedding_model),
using="sparse",
limit=20,
),
]
results = client.query_points(
collection_name,
prefetch=prefetch,
query=models.Document(text=query, model=late_interaction_embedding_model),
using="multi",
with_payload=True,
limit=10,
)
pprint.pp(results.points)
# @block-end rerank
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/rust.rs
+220
View File
@@ -0,0 +1,220 @@
use qdrant_client::Qdrant;
use qdrant_client::qdrant::{
CreateCollectionBuilder, Distance, Document, Fusion, HnswConfigDiffBuilder,
Modifier, MultiVectorComparator, MultiVectorConfigBuilder, NamedVectors, PointStruct,
PrefetchQueryBuilder, Query, QueryPointsBuilder, SparseVectorParamsBuilder,
SparseVectorsConfigBuilder, UpsertPointsBuilder, VectorParamsBuilder, VectorsConfigBuilder,
};
pub async fn main() -> anyhow::Result<()> {
// @hide-start
let qdrant_url = "https://xyz-example.eu-central.aws.cloud.qdrant.io:6334";
let qdrant_api_key = "<your-api-key>";
// @hide-end
// @block-start client-connection
let client = Qdrant::from_url(qdrant_url)
.api_key(qdrant_api_key)
.build()?;
// @block-end client-connection
// @block-start define-models
let dense_embedding_model = "sentence-transformers/all-MiniLM-L6-v2";
let sparse_embedding_model = "qdrant/bm25";
let late_interaction_embedding_model = "answerdotai/answerai-colbert-small-v1";
// @block-end define-models
// @block-start create-collection
let collection_name = "hybrid-search";
if client.collection_exists(collection_name).await? {
client.delete_collection(collection_name).await?;
}
let mut vectors = VectorsConfigBuilder::default();
vectors.add_named_vector_params(
"dense",
VectorParamsBuilder::new(384, Distance::Cosine),
);
vectors.add_named_vector_params(
"multi",
VectorParamsBuilder::new(96, Distance::Cosine)
.multivector_config(MultiVectorConfigBuilder::new(MultiVectorComparator::MaxSim))
.hnsw_config(HnswConfigDiffBuilder::default().m(0)), // Disable HNSW for reranking
);
let mut sparse = SparseVectorsConfigBuilder::default();
sparse.add_named_vector_params(
"sparse",
SparseVectorParamsBuilder::default().modifier(Modifier::Idf),
);
client
.create_collection(
CreateCollectionBuilder::new(collection_name)
.vectors_config(vectors)
.sparse_vectors_config(sparse),
)
.await?;
// @block-end create-collection
// @block-start parse-csv
struct CsvRow {
title: String,
author: String,
description: String,
}
fn parse_csv(url: &str) -> anyhow::Result<impl Iterator<Item = anyhow::Result<CsvRow>>> {
let reader = ureq::get(url).call()?.into_body().into_reader();
let mut rdr = csv::Reader::from_reader(reader);
let headers = rdr.headers()?.clone();
let title_idx = headers.iter().position(|h| h == "Title").unwrap();
let author_idx = headers.iter().position(|h| h == "Author").unwrap();
let description_idx = headers.iter().position(|h| h == "Description").unwrap();
let iter = rdr.into_records().map(move |result| {
let record = result?;
Ok(CsvRow {
title: record[title_idx].to_string(),
author: record[author_idx].to_string(),
description: record[description_idx].to_string(),
})
});
Ok(iter)
}
// @block-end parse-csv
// @block-start ingest-data
let csv_url = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/sci-fi-books/top_100_scifi_books_full.csv";
let batch_size = 25;
let mut idx: u64 = 0;
let mut buffer: Vec<PointStruct> = Vec::new();
for row in parse_csv(csv_url)? {
let row = row?;
let title = row.title;
let author = row.author;
let description = row.description;
let vectors = NamedVectors::default()
.add_vector("dense", Document::new(&description, dense_embedding_model))
.add_vector("sparse", Document::new(&description, sparse_embedding_model))
.add_vector("multi", Document::new(&description, late_interaction_embedding_model));
buffer.push(PointStruct::new(
idx,
vectors,
[
("title", title.into()),
("author", author.into()),
("description", description.into()),
],
));
idx += 1;
if buffer.len() >= batch_size {
client
.upsert_points(UpsertPointsBuilder::new(
collection_name,
std::mem::take(&mut buffer),
))
.await?;
}
}
if !buffer.is_empty() {
client
.upsert_points(UpsertPointsBuilder::new(collection_name, buffer))
.await?;
}
// @block-end ingest-data
// @block-start dense-retrieval
let query = "time travel";
let results = client
.query(
QueryPointsBuilder::new(collection_name)
.query(Query::new_nearest(Document::new(query, dense_embedding_model)))
.using("dense")
.limit(10),
)
.await?;
for result in results.result {
println!("{:?}", result);
}
// @block-end dense-retrieval
// @block-start sparse-retrieval
let results = client
.query(
QueryPointsBuilder::new(collection_name)
.query(Query::new_nearest(Document::new(query, sparse_embedding_model)))
.using("sparse")
.limit(10),
)
.await?;
for result in results.result {
println!("{:?}", result);
}
// @block-end sparse-retrieval
// @block-start hybrid-search
let results = client
.query(
QueryPointsBuilder::new(collection_name)
.add_prefetch(
PrefetchQueryBuilder::default()
.query(Query::new_nearest(Document::new(query, dense_embedding_model)))
.using("dense")
.limit(20u64),
)
.add_prefetch(
PrefetchQueryBuilder::default()
.query(Query::new_nearest(Document::new(query, sparse_embedding_model)))
.using("sparse")
.limit(20u64),
)
.query(Query::new_fusion(Fusion::Rrf))
.with_payload(true)
.limit(10),
)
.await?;
for result in results.result {
println!("{:?}", result);
}
// @block-end hybrid-search
// @block-start rerank
let results = client
.query(
QueryPointsBuilder::new(collection_name)
.add_prefetch(
PrefetchQueryBuilder::default()
.query(Query::new_nearest(Document::new(query, dense_embedding_model)))
.using("dense")
.limit(20u64),
)
.add_prefetch(
PrefetchQueryBuilder::default()
.query(Query::new_nearest(Document::new(query, sparse_embedding_model)))
.using("sparse")
.limit(20u64),
)
.query(Query::new_nearest(Document::new(query, late_interaction_embedding_model)))
.using("multi")
.with_payload(true)
.limit(10),
)
.await?;
for result in results.result {
println!("{:?}", result);
}
// @block-end rerank
Ok(())
}
qdrant-landing/content/documentation/headless/snippets/tutorial-reranking-hybrid-search/typescript.ts
+199
View File
@@ -0,0 +1,199 @@
import { QdrantClient, Schemas } from "@qdrant/js-client-rest";
// @hide-start
const QDRANT_URL = "https://xyz-example.eu-central.aws.cloud.qdrant.io";
const QDRANT_API_KEY = "<your-api-key>";
// @hide-end
// @block-start client-connection
const client = new QdrantClient({
url: QDRANT_URL,
apiKey: QDRANT_API_KEY,
});
// @block-end client-connection
// @block-start define-models
const denseEmbeddingModel = "sentence-transformers/all-MiniLM-L6-v2";
const sparseEmbeddingModel = "qdrant/bm25";
const lateInteractionEmbeddingModel = "answerdotai/answerai-colbert-small-v1";
// @block-end define-models
// @block-start create-collection
const collectionName = "hybrid-search";
if (await client.collectionExists(collectionName)) {
await client.deleteCollection(collectionName);
}
await client.createCollection(collectionName, {
vectors: {
dense: {
size: 384,
distance: "Cosine",
},
multi: {
size: 96,
distance: "Cosine",
multivector_config: { comparator: "max_sim" },
hnsw_config: { m: 0 }, // Disable HNSW for reranking
},
},
sparse_vectors: {
sparse: { modifier: "idf" },
},
});
// @block-end create-collection
// @block-start parse-csv
function parseCsvLine(line: string): string[] {
const fields: string[] = [];
let i = 0;
while (i < line.length) {
if (line[i] === '"') {
i++;
let field = "";
while (i < line.length) {
if (line[i] === '"' && line[i + 1] === '"') { field += '"'; i += 2; }
else if (line[i] === '"') { i++; break; }
else { field += line[i++]; }
}
fields.push(field);
if (line[i] === ",") i++;
} else {
const start = i;
while (i < line.length && line[i] !== ",") i++;
fields.push(line.slice(start, i));
if (i < line.length) i++;
}
}
return fields;
}
async function* parseCSV(url: string): AsyncGenerator<{ title: string; author: string; description: string }> {
const response = await fetch(url);
const reader = response.body!.getReader();
const decoder = new TextDecoder();
let remainder = "";
let headers: string[] | null = null;
let titleIdx = -1;
let authorIdx = -1;
let descriptionIdx = -1;
while (true) {
const { done, value } = await reader.read();
const chunk = done ? "" : decoder.decode(value, { stream: true });
const lines = (remainder + chunk).split("\n");
remainder = done ? "" : lines.pop()!;
for (const line of lines) {
if (!line.trim()) continue;
if (headers === null) {
headers = parseCsvLine(line);
titleIdx = headers.indexOf("Title");
authorIdx = headers.indexOf("Author");
descriptionIdx = headers.indexOf("Description");
continue;
}
const fields = parseCsvLine(line);
yield { title: fields[titleIdx], author: fields[authorIdx], description: fields[descriptionIdx] };
}
if (done) break;
}
}
// @block-end parse-csv
// @block-start ingest-data
const csvUrl = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/sci-fi-books/top_100_scifi_books_full.csv";
const batchSize = 25;
let idx = 0;
let buffer: Schemas["PointStruct"][] = [];
for await (const { title, author, description } of parseCSV(csvUrl)) {
buffer.push({
id: idx++,
vector: {
dense: { text: description, model: denseEmbeddingModel },
sparse: { text: description, model: sparseEmbeddingModel },
multi: { text: description, model: lateInteractionEmbeddingModel },
},
payload: { title, author, description },
});
if (buffer.length >= batchSize) {
await client.upsert(collectionName, { points: buffer });
buffer = [];
}
}
if (buffer.length > 0) {
await client.upsert(collectionName, { points: buffer });
}
// @block-end ingest-data
// @block-start dense-retrieval
const query = "time travel";
const denseResults = await client.query(collectionName, {
query: { text: query, model: denseEmbeddingModel },
using: "dense",
limit: 10,
});
console.log(denseResults.points);
// @block-end dense-retrieval
// @block-start sparse-retrieval
const sparseResults = await client.query(collectionName, {
query: { text: query, model: sparseEmbeddingModel },
using: "sparse",
limit: 10,
});
console.log(sparseResults.points);
// @block-end sparse-retrieval
// @block-start hybrid-search
const hybridResults = await client.query(collectionName, {
prefetch: [
{
query: { text: query, model: denseEmbeddingModel },
using: "dense",
limit: 20,
},
{
query: { text: query, model: sparseEmbeddingModel },
using: "sparse",
limit: 20,
},
],
query: { fusion: "rrf" },
with_payload: true,
limit: 10,
});
console.log(hybridResults.points);
// @block-end hybrid-search
// @block-start rerank
const rerankedResults = await client.query(collectionName, {
prefetch: [
{
query: { text: query, model: denseEmbeddingModel },
using: "dense",
limit: 20,
},
{
query: { text: query, model: sparseEmbeddingModel },
using: "sparse",
limit: 20,
},
],
query: { text: query, model: lateInteractionEmbeddingModel },
using: "multi",
with_payload: true,
limit: 10,
});
console.log(rerankedResults.points);
// @block-end rerank