Time based sharding tutorial (#1805)

* add snippets

* add snippets

* add snippets

* add snippets descriptions

* Initial commit

* Updated tutorial

* Expand on shard_number

* Add sentence about optimal batch size

* Show how to query multiple shards

* Small edits

* Fixes

* Add to landing page; adjust weight

* Clean up code snippets

* Don't set shard_key in code snippet

* Review feedback

* Update links to moved pages

* Add C# code snippets

* Add Go code snippets

* Add Java code snippets

* Add Rust code snippets

* Add TS code snippets

* Add note about changing collection settings

* Move CSV parsing to separate function

* Trigger Build

* Remove unused dependency

---------

Co-authored-by: Abdon Pijpelink <abdon.pijpelink@qdrant.com>
This commit is contained in:
Derrick Mwiti
2026-04-16 22:19:08 +02:00
committed by GitHub
co-authored by Abdon Pijpelink
parent 6d041d78ba
commit dc564bbab0
70 changed files with 4159 additions and 0 deletions
@@ -0,0 +1,238 @@
using System.Net.Http;
using Microsoft.VisualBasic.FileIO;
using Qdrant.Client;
using Qdrant.Client.Grpc;
public class Snippet
{
public static async Task Run()
{
// @hide-start
string QDRANT_URL = "";
string QDRANT_API_KEY = "";
// @hide-end
// @block-start initialize-client
var client = new QdrantClient(
host: QDRANT_URL,
https: true,
apiKey: QDRANT_API_KEY
);
// @block-end initialize-client
// @block-start create-collection
string collectionName = "my_collection";
if (await client.CollectionExistsAsync(collectionName))
await client.DeleteCollectionAsync(collectionName);
await client.CreateCollectionAsync(
collectionName: collectionName,
vectorsConfig: new VectorParamsMap
{
Map = {
["dense_vector"] = new VectorParams { Size = 384, Distance = Distance.Cosine }
}
},
shardingMethod: ShardingMethod.Custom
);
// @block-end create-collection
// @block-start parse-csv
async IAsyncEnumerable<(string text, string datetime)> ParseCsv(string url)
{
using var httpClient = new HttpClient();
using var stream = await httpClient.GetStreamAsync(url);
using var parser = new TextFieldParser(new StreamReader(stream));
parser.TextFieldType = Microsoft.VisualBasic.FileIO.FieldType.Delimited;
parser.SetDelimiters(",");
string[]? headers = parser.ReadFields();
int textIdx = Array.IndexOf(headers!, "text");
int datetimeIdx = Array.IndexOf(headers!, "datetime");
while (!parser.EndOfData)
{
var fields = parser.ReadFields()!;
yield return (fields[textIdx], fields[datetimeIdx]);
}
}
// @block-end parse-csv
// @block-start upload-vectors
string csvUrl = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv";
// Retrieve a list of existing shard keys in the collection
var existingShardKeys = (await client.ListShardKeysAsync(collectionName))
.Select(sk => sk.Key.Keyword)
.ToHashSet();
string denseModel = "sentence-transformers/all-MiniLM-L6-v2";
int batchSize = 100;
string? currentDate = null;
var buffer = new List<PointStruct>();
await foreach (var (text, datetime) in ParseCsv(csvUrl))
{
string shardDate = datetime[..10]; // Extract YYYY-MM-DD
if (shardDate != currentDate)
{
// Flush buffer for the previous date before switching
if (buffer.Count > 0)
{
await client.UpsertAsync(
collectionName: collectionName,
points: buffer,
shardKeySelector: new ShardKeySelector
{
ShardKeys = { new List<ShardKey> { currentDate! } }
}
);
buffer.Clear();
}
// Create shard for the new date if it doesn't exist yet
if (!existingShardKeys.Contains(shardDate))
{
await client.CreateShardKeyAsync(
collectionName,
new CreateShardKey { ShardKey = new ShardKey { Keyword = shardDate } }
);
existingShardKeys.Add(shardDate);
}
currentDate = shardDate;
}
// Add point to buffer
buffer.Add(new PointStruct
{
Id = Guid.NewGuid(),
Vectors = new Dictionary<string, Vector>
{
["dense_vector"] = new Document { Text = text, Model = denseModel }
},
Payload = { ["text"] = text, ["datetime"] = datetime }
});
// Flush batch if buffer size exceeds batch size
if (buffer.Count >= batchSize)
{
await client.UpsertAsync(
collectionName: collectionName,
points: buffer,
shardKeySelector: new ShardKeySelector
{
ShardKeys = { new List<ShardKey> { currentDate! } }
}
);
buffer.Clear();
}
}
// Flush remaining partial batch
if (buffer.Count > 0)
{
await client.UpsertAsync(
collectionName: collectionName,
points: buffer,
shardKeySelector: new ShardKeySelector
{
ShardKeys = { new List<ShardKey> { currentDate! } }
}
);
}
// @block-end upload-vectors
// @block-start search-single-shard
string queryText = "coffee";
var result = await client.QueryAsync(
collectionName: collectionName,
query: new Document { Text = queryText, Model = denseModel },
usingVector: "dense_vector",
limit: 5,
shardKeySelector: new ShardKeySelector
{
ShardKeys = { new List<ShardKey> { "2026-04-07" } }
}
);
foreach (var hit in result)
Console.WriteLine(hit);
// @block-end search-single-shard
// @block-start search-multiple-shards
result = await client.QueryAsync(
collectionName: collectionName,
query: new Document { Text = queryText, Model = denseModel },
usingVector: "dense_vector",
limit: 5,
shardKeySelector: new ShardKeySelector
{
ShardKeys = { new List<ShardKey> { "2026-04-06", "2026-04-07" } }
}
);
foreach (var hit in result)
Console.WriteLine(hit);
// @block-end search-multiple-shards
// @block-start search-all-shards
result = await client.QueryAsync(
collectionName: collectionName,
query: new Document { Text = queryText, Model = denseModel },
usingVector: "dense_vector",
limit: 5
);
foreach (var hit in result)
Console.WriteLine(hit);
// @block-end search-all-shards
// @block-start pruning-shards
string today = "2026-04-08";
string oldestShardKey = DateOnly.ParseExact(today, "yyyy-MM-dd")
.AddDays(-7)
.ToString("yyyy-MM-dd");
await client.CreateShardKeyAsync(
collectionName,
new CreateShardKey { ShardKey = new ShardKey { Keyword = today } }
);
await client.DeleteShardKeyAsync(
collectionName,
new DeleteShardKey { ShardKey = new ShardKey { Keyword = oldestShardKey } }
);
// @block-end pruning-shards
// @block-start ingest-new-data
await client.UpsertAsync(
collectionName: collectionName,
points: new List<PointStruct>
{
new()
{
Id = Guid.NewGuid(),
Vectors = new Dictionary<string, Vector>
{
["dense_vector"] = new Document
{
Text = "The best way to start a Wednesday is with a cup of coffee",
Model = denseModel
}
},
Payload =
{
["text"] = "The best way to start a Wednesday is with a cup of coffee",
["datetime"] = "2026-04-08T07:57:47"
}
}
},
shardKeySelector: new ShardKeySelector
{
ShardKeys = { new List<ShardKey> { today } }
}
);
// @block-end ingest-new-data
}
}
@@ -0,0 +1,17 @@
```csharp
string collectionName = "my_collection";
if (await client.CollectionExistsAsync(collectionName))
await client.DeleteCollectionAsync(collectionName);
await client.CreateCollectionAsync(
collectionName: collectionName,
vectorsConfig: new VectorParamsMap
{
Map = {
["dense_vector"] = new VectorParams { Size = 384, Distance = Distance.Cosine }
}
},
shardingMethod: ShardingMethod.Custom
);
```
@@ -0,0 +1,21 @@
```go
collectionName := "my_collection"
exists, err := client.CollectionExists(context.Background(), collectionName)
if exists {
client.DeleteCollection(context.Background(), collectionName)
}
client.CreateCollection(context.Background(), &qdrant.CreateCollection{
CollectionName: collectionName,
VectorsConfig: qdrant.NewVectorsConfigMap(
map[string]*qdrant.VectorParams{
"dense_vector": {
Size: 384,
Distance: qdrant.Distance_Cosine,
},
},
),
ShardingMethod: qdrant.ShardingMethod_Custom.Enum(),
})
```
@@ -0,0 +1,21 @@
```java
String collectionName = "my_collection";
if (client.collectionExistsAsync(collectionName).get()) {
client.deleteCollectionAsync(collectionName).get();
}
client.createCollectionAsync(
CreateCollection.newBuilder()
.setCollectionName(collectionName)
.setVectorsConfig(VectorsConfig.newBuilder().setParamsMap(
VectorParamsMap.newBuilder().putAllMap(Map.of(
"dense_vector",
VectorParams.newBuilder()
.setSize(384)
.setDistance(Distance.Cosine)
.build()))))
.setShardingMethod(ShardingMethod.Custom)
.build()
).get();
```
@@ -0,0 +1,18 @@
```python
from qdrant_client import models
collection_name = "my_collection"
if client.collection_exists(collection_name=collection_name):
client.delete_collection(collection_name=collection_name)
client.create_collection(
collection_name=collection_name,
vectors_config={
"dense_vector": models.VectorParams(
size=384, distance=models.Distance.COSINE
)
},
sharding_method=models.ShardingMethod.CUSTOM
)
```
@@ -0,0 +1,21 @@
```rust
let collection_name = "my_collection";
if client.collection_exists(collection_name).await? {
client.delete_collection(collection_name).await?;
}
let mut vectors_config = VectorsConfigBuilder::default();
vectors_config.add_named_vector_params(
"dense_vector",
VectorParamsBuilder::new(384, Distance::Cosine),
);
client
.create_collection(
CreateCollectionBuilder::new(collection_name)
.vectors_config(vectors_config)
.sharding_method(ShardingMethod::Custom.into()),
)
.await?;
```
@@ -0,0 +1,17 @@
```typescript
const collectionName = "my_collection";
if (await client.collectionExists(collectionName)) {
await client.deleteCollection(collectionName);
}
await client.createCollection(collectionName, {
vectors: {
dense_vector: {
size: 384,
distance: "Cosine",
},
},
sharding_method: "custom",
});
```
@@ -0,0 +1,211 @@
```csharp
using System.Net.Http;
using Microsoft.VisualBasic.FileIO;
using Qdrant.Client;
using Qdrant.Client.Grpc;
var client = new QdrantClient(
host: QDRANT_URL,
https: true,
apiKey: QDRANT_API_KEY
);
string collectionName = "my_collection";
if (await client.CollectionExistsAsync(collectionName))
await client.DeleteCollectionAsync(collectionName);
await client.CreateCollectionAsync(
collectionName: collectionName,
vectorsConfig: new VectorParamsMap
{
Map = {
["dense_vector"] = new VectorParams { Size = 384, Distance = Distance.Cosine }
}
},
shardingMethod: ShardingMethod.Custom
);
async IAsyncEnumerable<(string text, string datetime)> ParseCsv(string url)
{
using var httpClient = new HttpClient();
using var stream = await httpClient.GetStreamAsync(url);
using var parser = new TextFieldParser(new StreamReader(stream));
parser.TextFieldType = Microsoft.VisualBasic.FileIO.FieldType.Delimited;
parser.SetDelimiters(",");
string[]? headers = parser.ReadFields();
int textIdx = Array.IndexOf(headers!, "text");
int datetimeIdx = Array.IndexOf(headers!, "datetime");
while (!parser.EndOfData)
{
var fields = parser.ReadFields()!;
yield return (fields[textIdx], fields[datetimeIdx]);
}
}
string csvUrl = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv";
// Retrieve a list of existing shard keys in the collection
var existingShardKeys = (await client.ListShardKeysAsync(collectionName))
.Select(sk => sk.Key.Keyword)
.ToHashSet();
string denseModel = "sentence-transformers/all-MiniLM-L6-v2";
int batchSize = 100;
string? currentDate = null;
var buffer = new List<PointStruct>();
await foreach (var (text, datetime) in ParseCsv(csvUrl))
{
string shardDate = datetime[..10]; // Extract YYYY-MM-DD
if (shardDate != currentDate)
{
// Flush buffer for the previous date before switching
if (buffer.Count > 0)
{
await client.UpsertAsync(
collectionName: collectionName,
points: buffer,
shardKeySelector: new ShardKeySelector
{
ShardKeys = { new List<ShardKey> { currentDate! } }
}
);
buffer.Clear();
}
// Create shard for the new date if it doesn't exist yet
if (!existingShardKeys.Contains(shardDate))
{
await client.CreateShardKeyAsync(
collectionName,
new CreateShardKey { ShardKey = new ShardKey { Keyword = shardDate } }
);
existingShardKeys.Add(shardDate);
}
currentDate = shardDate;
}
// Add point to buffer
buffer.Add(new PointStruct
{
Id = Guid.NewGuid(),
Vectors = new Dictionary<string, Vector>
{
["dense_vector"] = new Document { Text = text, Model = denseModel }
},
Payload = { ["text"] = text, ["datetime"] = datetime }
});
// Flush batch if buffer size exceeds batch size
if (buffer.Count >= batchSize)
{
await client.UpsertAsync(
collectionName: collectionName,
points: buffer,
shardKeySelector: new ShardKeySelector
{
ShardKeys = { new List<ShardKey> { currentDate! } }
}
);
buffer.Clear();
}
}
// Flush remaining partial batch
if (buffer.Count > 0)
{
await client.UpsertAsync(
collectionName: collectionName,
points: buffer,
shardKeySelector: new ShardKeySelector
{
ShardKeys = { new List<ShardKey> { currentDate! } }
}
);
}
string queryText = "coffee";
var result = await client.QueryAsync(
collectionName: collectionName,
query: new Document { Text = queryText, Model = denseModel },
usingVector: "dense_vector",
limit: 5,
shardKeySelector: new ShardKeySelector
{
ShardKeys = { new List<ShardKey> { "2026-04-07" } }
}
);
foreach (var hit in result)
Console.WriteLine(hit);
result = await client.QueryAsync(
collectionName: collectionName,
query: new Document { Text = queryText, Model = denseModel },
usingVector: "dense_vector",
limit: 5,
shardKeySelector: new ShardKeySelector
{
ShardKeys = { new List<ShardKey> { "2026-04-06", "2026-04-07" } }
}
);
foreach (var hit in result)
Console.WriteLine(hit);
result = await client.QueryAsync(
collectionName: collectionName,
query: new Document { Text = queryText, Model = denseModel },
usingVector: "dense_vector",
limit: 5
);
foreach (var hit in result)
Console.WriteLine(hit);
string today = "2026-04-08";
string oldestShardKey = DateOnly.ParseExact(today, "yyyy-MM-dd")
.AddDays(-7)
.ToString("yyyy-MM-dd");
await client.CreateShardKeyAsync(
collectionName,
new CreateShardKey { ShardKey = new ShardKey { Keyword = today } }
);
await client.DeleteShardKeyAsync(
collectionName,
new DeleteShardKey { ShardKey = new ShardKey { Keyword = oldestShardKey } }
);
await client.UpsertAsync(
collectionName: collectionName,
points: new List<PointStruct>
{
new()
{
Id = Guid.NewGuid(),
Vectors = new Dictionary<string, Vector>
{
["dense_vector"] = new Document
{
Text = "The best way to start a Wednesday is with a cup of coffee",
Model = denseModel
}
},
Payload =
{
["text"] = "The best way to start a Wednesday is with a cup of coffee",
["datetime"] = "2026-04-08T07:57:47"
}
}
},
shardKeySelector: new ShardKeySelector
{
ShardKeys = { new List<ShardKey> { today } }
}
);
```
@@ -0,0 +1,240 @@
```go
import (
"context"
"encoding/csv"
"fmt"
"io"
"net/http"
"time"
"github.com/google/uuid"
"github.com/qdrant/go-client/qdrant"
)
type CSVRow struct {
Text string
Datetime string
}
func parseCSV(url string, fn func(CSVRow)) error {
resp, err := http.Get(url)
if err != nil {
return err
}
defer resp.Body.Close()
csvReader := csv.NewReader(resp.Body)
headers, err := csvReader.Read()
if err != nil {
return err
}
textIdx, datetimeIdx := -1, -1
for i, h := range headers {
switch h {
case "text":
textIdx = i
case "datetime":
datetimeIdx = i
}
}
for {
row, err := csvReader.Read()
if err == io.EOF {
break
}
if err != nil {
return err
}
fn(CSVRow{Text: row[textIdx], Datetime: row[datetimeIdx]})
}
return nil
}
client, err := qdrant.NewClient(&qdrant.Config{
Host: QDRANT_URL,
APIKey: QDRANT_API_KEY,
UseTLS: true,
})
collectionName := "my_collection"
exists, err := client.CollectionExists(context.Background(), collectionName)
if exists {
client.DeleteCollection(context.Background(), collectionName)
}
client.CreateCollection(context.Background(), &qdrant.CreateCollection{
CollectionName: collectionName,
VectorsConfig: qdrant.NewVectorsConfigMap(
map[string]*qdrant.VectorParams{
"dense_vector": {
Size: 384,
Distance: qdrant.Distance_Cosine,
},
},
),
ShardingMethod: qdrant.ShardingMethod_Custom.Enum(),
})
csvUrl := "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv"
shardKeyDescriptions, err := client.ListShardKeys(context.Background(), collectionName)
// Retrieve a list of existing shard keys in the collection
existingShardKeys := make(map[string]bool)
for _, desc := range shardKeyDescriptions {
existingShardKeys[desc.Key.GetKeyword()] = true
}
denseModel := "sentence-transformers/all-MiniLM-L6-v2"
batchSize := 100
var currentDate string
var buffer []*qdrant.PointStruct
err = parseCSV(csvUrl, func(row CSVRow) {
text := row.Text
datetime := row.Datetime
shardDate := datetime[:10] // Extract YYYY-MM-DD
if shardDate != currentDate {
// Flush buffer for the previous date before switching
if len(buffer) > 0 {
client.Upsert(context.Background(), &qdrant.UpsertPoints{
CollectionName: collectionName,
Points: buffer,
ShardKeySelector: &qdrant.ShardKeySelector{
ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey(currentDate)},
},
})
buffer = nil
}
// Create shard for the new date if it doesn't exist yet
if !existingShardKeys[shardDate] {
client.CreateShardKey(context.Background(), collectionName, &qdrant.CreateShardKey{
ShardKey: qdrant.NewShardKey(shardDate),
})
existingShardKeys[shardDate] = true
}
currentDate = shardDate
}
// Add point to buffer
buffer = append(buffer, &qdrant.PointStruct{
Id: qdrant.NewID(uuid.New().String()),
Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{
"dense_vector": qdrant.NewVectorDocument(&qdrant.Document{
Text: text,
Model: denseModel,
}),
}),
Payload: qdrant.NewValueMap(map[string]any{
"text": text,
"datetime": datetime,
}),
})
// Flush batch if buffer size exceeds batch size
if len(buffer) >= batchSize {
client.Upsert(context.Background(), &qdrant.UpsertPoints{
CollectionName: collectionName,
Points: buffer,
ShardKeySelector: &qdrant.ShardKeySelector{
ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey(currentDate)},
},
})
buffer = nil
}
})
// Flush remaining partial batch
if len(buffer) > 0 {
client.Upsert(context.Background(), &qdrant.UpsertPoints{
CollectionName: collectionName,
Points: buffer,
ShardKeySelector: &qdrant.ShardKeySelector{
ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey(currentDate)},
},
})
}
queryText := "coffee"
result, err := client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: collectionName,
Query: qdrant.NewQueryDocument(&qdrant.Document{Text: queryText, Model: denseModel}),
Using: qdrant.PtrOf("dense_vector"),
Limit: qdrant.PtrOf(uint64(5)),
ShardKeySelector: &qdrant.ShardKeySelector{
ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey("2026-04-07")},
},
})
for _, hit := range result {
fmt.Println(hit)
}
result, err = client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: collectionName,
Query: qdrant.NewQueryDocument(&qdrant.Document{Text: queryText, Model: denseModel}),
Using: qdrant.PtrOf("dense_vector"),
Limit: qdrant.PtrOf(uint64(5)),
ShardKeySelector: &qdrant.ShardKeySelector{
ShardKeys: []*qdrant.ShardKey{
qdrant.NewShardKey("2026-04-06"),
qdrant.NewShardKey("2026-04-07"),
},
},
})
for _, hit := range result {
fmt.Println(hit)
}
result, err = client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: collectionName,
Query: qdrant.NewQueryDocument(&qdrant.Document{Text: queryText, Model: denseModel}),
Using: qdrant.PtrOf("dense_vector"),
Limit: qdrant.PtrOf(uint64(5)),
})
for _, hit := range result {
fmt.Println(hit)
}
today := "2026-04-08"
t, _ := time.Parse("2006-01-02", today)
oldestShardKey := t.AddDate(0, 0, -7).Format("2006-01-02")
client.CreateShardKey(context.Background(), collectionName, &qdrant.CreateShardKey{
ShardKey: qdrant.NewShardKey(today),
})
client.DeleteShardKey(context.Background(), collectionName, &qdrant.DeleteShardKey{
ShardKey: qdrant.NewShardKey(oldestShardKey),
})
client.Upsert(context.Background(), &qdrant.UpsertPoints{
CollectionName: collectionName,
Points: []*qdrant.PointStruct{
{
Id: qdrant.NewID(uuid.New().String()),
Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{
"dense_vector": qdrant.NewVectorDocument(&qdrant.Document{
Text: "The best way to start a Wednesday is with a cup of coffee",
Model: denseModel,
}),
}),
Payload: qdrant.NewValueMap(map[string]any{
"text": "The best way to start a Wednesday is with a cup of coffee",
"datetime": "2026-04-08T07:57:47",
}),
},
},
ShardKeySelector: &qdrant.ShardKeySelector{
ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey(today)},
},
})
```
@@ -0,0 +1,29 @@
```csharp
await client.UpsertAsync(
collectionName: collectionName,
points: new List<PointStruct>
{
new()
{
Id = Guid.NewGuid(),
Vectors = new Dictionary<string, Vector>
{
["dense_vector"] = new Document
{
Text = "The best way to start a Wednesday is with a cup of coffee",
Model = denseModel
}
},
Payload =
{
["text"] = "The best way to start a Wednesday is with a cup of coffee",
["datetime"] = "2026-04-08T07:57:47"
}
}
},
shardKeySelector: new ShardKeySelector
{
ShardKeys = { new List<ShardKey> { today } }
}
);
```
@@ -0,0 +1,23 @@
```go
client.Upsert(context.Background(), &qdrant.UpsertPoints{
CollectionName: collectionName,
Points: []*qdrant.PointStruct{
{
Id: qdrant.NewID(uuid.New().String()),
Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{
"dense_vector": qdrant.NewVectorDocument(&qdrant.Document{
Text: "The best way to start a Wednesday is with a cup of coffee",
Model: denseModel,
}),
}),
Payload: qdrant.NewValueMap(map[string]any{
"text": "The best way to start a Wednesday is with a cup of coffee",
"datetime": "2026-04-08T07:57:47",
}),
},
},
ShardKeySelector: &qdrant.ShardKeySelector{
ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey(today)},
},
})
```
@@ -0,0 +1,21 @@
```java
client.upsertAsync(
UpsertPoints.newBuilder()
.setCollectionName(collectionName)
.addAllPoints(List.of(
PointStruct.newBuilder()
.setId(id(UUID.randomUUID()))
.setVectors(namedVectors(Map.of(
"dense_vector",
vector(Document.newBuilder()
.setText("The best way to start a Wednesday is with a cup of coffee")
.setModel(denseModel)
.build()))))
.putAllPayload(Map.of(
"text", value("The best way to start a Wednesday is with a cup of coffee"),
"datetime", value("2026-04-08T07:57:47")))
.build()))
.setShardKeySelector(shardKeySelector(today))
.build()
).get();
```
@@ -0,0 +1,12 @@
```python
client.upsert(
collection_name=collection_name,
points=[PointStruct(
id=uuid.uuid4().hex,
payload={"text": "The best way to start a Wednesday is with a cup of coffee", "datetime": "2026-04-08T07:57:47"},
vector={
"dense_vector": Document(text="The best way to start a Wednesday is with a cup of coffee", model=dense_model)
})],
shard_key_selector=today
)
```
@@ -0,0 +1,25 @@
```rust
client
.upsert_points(
UpsertPointsBuilder::new(
collection_name,
vec![PointStruct::new(
uuid::Uuid::new_v4().to_string(),
HashMap::from([(
"dense_vector".to_string(),
DocumentBuilder::new(
"The best way to start a Wednesday is with a cup of coffee",
dense_model,
)
.build(),
)]),
[
("text", "The best way to start a Wednesday is with a cup of coffee".into()),
("datetime", "2026-04-08T07:57:47".into()),
],
)],
)
.shard_key_selector(today.to_string()),
)
.await?;
```
@@ -0,0 +1,20 @@
```typescript
await client.upsert(collectionName, {
points: [
{
id: crypto.randomUUID(),
vector: {
dense_vector: {
text: "The best way to start a Wednesday is with a cup of coffee",
model: denseModel,
},
},
payload: {
text: "The best way to start a Wednesday is with a cup of coffee",
datetime: "2026-04-08T07:57:47",
},
},
],
shard_key: today,
});
```
@@ -0,0 +1,7 @@
```csharp
var client = new QdrantClient(
host: QDRANT_URL,
https: true,
apiKey: QDRANT_API_KEY
);
```
@@ -0,0 +1,7 @@
```go
client, err := qdrant.NewClient(&qdrant.Config{
Host: QDRANT_URL,
APIKey: QDRANT_API_KEY,
UseTLS: true,
})
```
@@ -0,0 +1,7 @@
```java
QdrantClient client =
new QdrantClient(
QdrantGrpcClient.newBuilder(QDRANT_URL, 6334, true)
.withApiKey(QDRANT_API_KEY)
.build());
```
@@ -0,0 +1,9 @@
```python
from qdrant_client import QdrantClient, models
client = QdrantClient(
url=QDRANT_URL,
api_key=QDRANT_API_KEY,
cloud_inference=True
)
```
@@ -0,0 +1,5 @@
```rust
let client = Qdrant::from_url(QDRANT_URL)
.api_key(QDRANT_API_KEY)
.build()?;
```
@@ -0,0 +1,6 @@
```typescript
const client = new QdrantClient({
url: QDRANT_URL,
apiKey: QDRANT_API_KEY,
});
```
@@ -0,0 +1,278 @@
```java
import static io.qdrant.client.PointIdFactory.id;
import static io.qdrant.client.QueryFactory.nearest;
import static io.qdrant.client.ShardKeyFactory.shardKey;
import static io.qdrant.client.ShardKeySelectorFactory.shardKeySelector;
import static io.qdrant.client.ValueFactory.value;
import static io.qdrant.client.VectorFactory.vector;
import static io.qdrant.client.VectorsFactory.namedVectors;
import io.qdrant.client.QdrantClient;
import io.qdrant.client.QdrantGrpcClient;
import io.qdrant.client.grpc.Collections.CreateCollection;
import io.qdrant.client.grpc.Collections.CreateShardKey;
import io.qdrant.client.grpc.Collections.CreateShardKeyRequest;
import io.qdrant.client.grpc.Collections.DeleteShardKey;
import io.qdrant.client.grpc.Collections.DeleteShardKeyRequest;
import io.qdrant.client.grpc.Collections.Distance;
import io.qdrant.client.grpc.Collections.ShardingMethod;
import io.qdrant.client.grpc.Collections.VectorParams;
import io.qdrant.client.grpc.Collections.VectorParamsMap;
import io.qdrant.client.grpc.Collections.VectorsConfig;
import io.qdrant.client.grpc.Points.Document;
import io.qdrant.client.grpc.Points.PointStruct;
import io.qdrant.client.grpc.Points.QueryPoints;
import io.qdrant.client.grpc.Points.ShardKeySelector;
import io.qdrant.client.grpc.Points.UpsertPoints;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.URL;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.function.Function;
import java.util.stream.Stream;
static class CsvRow {
final String text;
final String datetime;
CsvRow(String text, String datetime) { this.text = text; this.datetime = datetime; }
}
static Stream<CsvRow> parseCSV(String url) throws Exception {
Function<String, List<String>> parseCsvLine = line -> {
List<String> fields = new ArrayList<>();
boolean inQuotes = false;
var sb = new StringBuilder();
for (char c : line.toCharArray()) {
if (c == '"') {
inQuotes = !inQuotes;
} else if (c == ',' && !inQuotes) {
fields.add(sb.toString());
sb.setLength(0);
} else {
sb.append(c);
}
}
fields.add(sb.toString());
return fields;
};
var reader = new BufferedReader(new InputStreamReader(new URL(url).openStream()));
String headerLine = reader.readLine();
List<String> headers = List.of(headerLine.split(","));
int textIdx = headers.indexOf("text");
int datetimeIdx = headers.indexOf("datetime");
return reader.lines()
.map(line -> {
List<String> fields = parseCsvLine.apply(line);
return new CsvRow(fields.get(textIdx), fields.get(datetimeIdx));
})
.onClose(() -> { try { reader.close(); } catch (Exception ignored) {} });
}
QdrantClient client =
new QdrantClient(
QdrantGrpcClient.newBuilder(QDRANT_URL, 6334, true)
.withApiKey(QDRANT_API_KEY)
.build());
String collectionName = "my_collection";
if (client.collectionExistsAsync(collectionName).get()) {
client.deleteCollectionAsync(collectionName).get();
}
client.createCollectionAsync(
CreateCollection.newBuilder()
.setCollectionName(collectionName)
.setVectorsConfig(VectorsConfig.newBuilder().setParamsMap(
VectorParamsMap.newBuilder().putAllMap(Map.of(
"dense_vector",
VectorParams.newBuilder()
.setSize(384)
.setDistance(Distance.Cosine)
.build()))))
.setShardingMethod(ShardingMethod.Custom)
.build()
).get();
String csvUrl = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv";
// Retrieve a list of existing shard keys in the collection
var shardKeyDescriptions = client.listShardKeysAsync(collectionName).get();
Set<String> existingShardKeys = new HashSet<>();
for (var desc : shardKeyDescriptions) {
existingShardKeys.add(desc.getKey().getKeyword());
}
String denseModel = "sentence-transformers/all-MiniLM-L6-v2";
int batchSize = 100;
String currentDate = null;
List<PointStruct> buffer = new ArrayList<>();
try (var stream = parseCSV(csvUrl)) {
for (var row : (Iterable<CsvRow>) stream::iterator) {
String text = row.text;
String datetime = row.datetime;
String shardDate = datetime.substring(0, 10); // Extract YYYY-MM-DD
if (!shardDate.equals(currentDate)) {
// Flush buffer for the previous date before switching
if (!buffer.isEmpty()) {
client.upsertAsync(
UpsertPoints.newBuilder()
.setCollectionName(collectionName)
.addAllPoints(buffer)
.setShardKeySelector(shardKeySelector(currentDate))
.build()
).get();
buffer.clear();
}
// Create shard for the new date if it doesn't exist yet
if (!existingShardKeys.contains(shardDate)) {
client.createShardKeyAsync(
CreateShardKeyRequest.newBuilder()
.setCollectionName(collectionName)
.setRequest(CreateShardKey.newBuilder()
.setShardKey(shardKey(shardDate))
.build())
.build()
).get();
existingShardKeys.add(shardDate);
}
currentDate = shardDate;
}
// Add point to buffer
buffer.add(
PointStruct.newBuilder()
.setId(id(UUID.randomUUID()))
.setVectors(namedVectors(Map.of(
"dense_vector",
vector(Document.newBuilder()
.setText(text)
.setModel(denseModel)
.build()))))
.putAllPayload(Map.of("text", value(text), "datetime", value(datetime)))
.build());
// Flush batch if buffer size exceeds batch size
if (buffer.size() >= batchSize) {
client.upsertAsync(
UpsertPoints.newBuilder()
.setCollectionName(collectionName)
.addAllPoints(buffer)
.setShardKeySelector(shardKeySelector(currentDate))
.build()
).get();
buffer.clear();
}
}
}
// Flush remaining partial batch
if (!buffer.isEmpty()) {
client.upsertAsync(
UpsertPoints.newBuilder()
.setCollectionName(collectionName)
.addAllPoints(buffer)
.setShardKeySelector(shardKeySelector(currentDate))
.build()
).get();
}
String queryText = "coffee";
var result = client.queryAsync(
QueryPoints.newBuilder()
.setCollectionName(collectionName)
.setQuery(nearest(Document.newBuilder().setText(queryText).setModel(denseModel).build()))
.setUsing("dense_vector")
.setLimit(5)
.setShardKeySelector(shardKeySelector("2026-04-07"))
.build()
).get();
for (var hit : result) {
System.out.println(hit);
}
result = client.queryAsync(
QueryPoints.newBuilder()
.setCollectionName(collectionName)
.setQuery(nearest(Document.newBuilder().setText(queryText).setModel(denseModel).build()))
.setUsing("dense_vector")
.setLimit(5)
.setShardKeySelector(ShardKeySelector.newBuilder()
.addShardKeys(shardKey("2026-04-06"))
.addShardKeys(shardKey("2026-04-07"))
.build())
.build()
).get();
for (var hit : result) {
System.out.println(hit);
}
result = client.queryAsync(
QueryPoints.newBuilder()
.setCollectionName(collectionName)
.setQuery(nearest(Document.newBuilder().setText(queryText).setModel(denseModel).build()))
.setUsing("dense_vector")
.setLimit(5)
.build()
).get();
for (var hit : result) {
System.out.println(hit);
}
String today = "2026-04-08";
String oldestShardKey = LocalDate.parse(today).minusDays(7).toString();
client.createShardKeyAsync(
CreateShardKeyRequest.newBuilder()
.setCollectionName(collectionName)
.setRequest(CreateShardKey.newBuilder()
.setShardKey(shardKey(today))
.build())
.build()
).get();
client.deleteShardKeyAsync(
DeleteShardKeyRequest.newBuilder()
.setCollectionName(collectionName)
.setRequest(DeleteShardKey.newBuilder()
.setShardKey(shardKey(oldestShardKey))
.build())
.build()
).get();
client.upsertAsync(
UpsertPoints.newBuilder()
.setCollectionName(collectionName)
.addAllPoints(List.of(
PointStruct.newBuilder()
.setId(id(UUID.randomUUID()))
.setVectors(namedVectors(Map.of(
"dense_vector",
vector(Document.newBuilder()
.setText("The best way to start a Wednesday is with a cup of coffee")
.setModel(denseModel)
.build()))))
.putAllPayload(Map.of(
"text", value("The best way to start a Wednesday is with a cup of coffee"),
"datetime", value("2026-04-08T07:57:47")))
.build()))
.setShardKeySelector(shardKeySelector(today))
.build()
).get();
```
@@ -0,0 +1,18 @@
```csharp
async IAsyncEnumerable<(string text, string datetime)> ParseCsv(string url)
{
using var httpClient = new HttpClient();
using var stream = await httpClient.GetStreamAsync(url);
using var parser = new TextFieldParser(new StreamReader(stream));
parser.TextFieldType = Microsoft.VisualBasic.FileIO.FieldType.Delimited;
parser.SetDelimiters(",");
string[]? headers = parser.ReadFields();
int textIdx = Array.IndexOf(headers!, "text");
int datetimeIdx = Array.IndexOf(headers!, "datetime");
while (!parser.EndOfData)
{
var fields = parser.ReadFields()!;
yield return (fields[textIdx], fields[datetimeIdx]);
}
}
```
@@ -0,0 +1,42 @@
```go
type CSVRow struct {
Text string
Datetime string
}
func parseCSV(url string, fn func(CSVRow)) error {
resp, err := http.Get(url)
if err != nil {
return err
}
defer resp.Body.Close()
csvReader := csv.NewReader(resp.Body)
headers, err := csvReader.Read()
if err != nil {
return err
}
textIdx, datetimeIdx := -1, -1
for i, h := range headers {
switch h {
case "text":
textIdx = i
case "datetime":
datetimeIdx = i
}
}
for {
row, err := csvReader.Read()
if err == io.EOF {
break
}
if err != nil {
return err
}
fn(CSVRow{Text: row[textIdx], Datetime: row[datetimeIdx]})
}
return nil
}
```
@@ -0,0 +1,40 @@
```java
static class CsvRow {
final String text;
final String datetime;
CsvRow(String text, String datetime) { this.text = text; this.datetime = datetime; }
}
static Stream<CsvRow> parseCSV(String url) throws Exception {
Function<String, List<String>> parseCsvLine = line -> {
List<String> fields = new ArrayList<>();
boolean inQuotes = false;
var sb = new StringBuilder();
for (char c : line.toCharArray()) {
if (c == '"') {
inQuotes = !inQuotes;
} else if (c == ',' && !inQuotes) {
fields.add(sb.toString());
sb.setLength(0);
} else {
sb.append(c);
}
}
fields.add(sb.toString());
return fields;
};
var reader = new BufferedReader(new InputStreamReader(new URL(url).openStream()));
String headerLine = reader.readLine();
List<String> headers = List.of(headerLine.split(","));
int textIdx = headers.indexOf("text");
int datetimeIdx = headers.indexOf("datetime");
return reader.lines()
.map(line -> {
List<String> fields = parseCsvLine.apply(line);
return new CsvRow(fields.get(textIdx), fields.get(datetimeIdx));
})
.onClose(() -> { try { reader.close(); } catch (Exception ignored) {} });
}
```
@@ -0,0 +1,9 @@
```python
import csv
import urllib.request
def parse_csv(url):
with urllib.request.urlopen(url) as response:
reader = csv.DictReader(line.decode('utf-8') for line in response)
yield from reader
```
@@ -0,0 +1,22 @@
```rust
struct CsvRow {
text: String,
datetime: String,
}
fn parse_csv(url: &str) -> anyhow::Result<impl Iterator<Item = anyhow::Result<CsvRow>>> {
let reader = ureq::get(url).call()?.into_body().into_reader();
let mut rdr = csv::Reader::from_reader(reader);
let headers = rdr.headers()?.clone();
let text_idx = headers.iter().position(|h| h == "text").unwrap();
let datetime_idx = headers.iter().position(|h| h == "datetime").unwrap();
let iter = rdr.into_records().map(move |result| {
let record = result?;
Ok(CsvRow {
text: record[text_idx].to_string(),
datetime: record[datetime_idx].to_string(),
})
});
Ok(iter)
}
```
@@ -0,0 +1,56 @@
```typescript
function parseCsvLine(line: string): string[] {
const fields: string[] = [];
let i = 0;
while (i < line.length) {
if (line[i] === '"') {
i++;
let field = "";
while (i < line.length) {
if (line[i] === '"' && line[i + 1] === '"') { field += '"'; i += 2; }
else if (line[i] === '"') { i++; break; }
else { field += line[i++]; }
}
fields.push(field);
if (line[i] === ",") i++;
} else {
const start = i;
while (i < line.length && line[i] !== ",") i++;
fields.push(line.slice(start, i));
if (i < line.length) i++;
}
}
return fields;
}
async function* parseCSV(url: string): AsyncGenerator<{ text: string; datetime: string }> {
const response = await fetch(url);
const reader = response.body!.getReader();
const decoder = new TextDecoder();
let remainder = "";
let headers: string[] | null = null;
let textIdx = -1;
let datetimeIdx = -1;
while (true) {
const { done, value } = await reader.read();
const chunk = done ? "" : decoder.decode(value, { stream: true });
const lines = (remainder + chunk).split("\n");
remainder = done ? "" : lines.pop()!;
for (const line of lines) {
if (!line.trim()) continue;
if (headers === null) {
headers = line.split(",");
textIdx = headers.indexOf("text");
datetimeIdx = headers.indexOf("datetime");
continue;
}
const fields = parseCsvLine(line);
yield { text: fields[textIdx], datetime: fields[datetimeIdx] };
}
if (done) break;
}
}
```
@@ -0,0 +1,15 @@
```csharp
string today = "2026-04-08";
string oldestShardKey = DateOnly.ParseExact(today, "yyyy-MM-dd")
.AddDays(-7)
.ToString("yyyy-MM-dd");
await client.CreateShardKeyAsync(
collectionName,
new CreateShardKey { ShardKey = new ShardKey { Keyword = today } }
);
await client.DeleteShardKeyAsync(
collectionName,
new DeleteShardKey { ShardKey = new ShardKey { Keyword = oldestShardKey } }
);
```
@@ -0,0 +1,12 @@
```go
today := "2026-04-08"
t, _ := time.Parse("2006-01-02", today)
oldestShardKey := t.AddDate(0, 0, -7).Format("2006-01-02")
client.CreateShardKey(context.Background(), collectionName, &qdrant.CreateShardKey{
ShardKey: qdrant.NewShardKey(today),
})
client.DeleteShardKey(context.Background(), collectionName, &qdrant.DeleteShardKey{
ShardKey: qdrant.NewShardKey(oldestShardKey),
})
```
@@ -0,0 +1,22 @@
```java
String today = "2026-04-08";
String oldestShardKey = LocalDate.parse(today).minusDays(7).toString();
client.createShardKeyAsync(
CreateShardKeyRequest.newBuilder()
.setCollectionName(collectionName)
.setRequest(CreateShardKey.newBuilder()
.setShardKey(shardKey(today))
.build())
.build()
).get();
client.deleteShardKeyAsync(
DeleteShardKeyRequest.newBuilder()
.setCollectionName(collectionName)
.setRequest(DeleteShardKey.newBuilder()
.setShardKey(shardKey(oldestShardKey))
.build())
.build()
).get();
```
@@ -0,0 +1,9 @@
```python
from datetime import date, timedelta
today = "2026-04-08"
oldest_shard_key = (date.fromisoformat(today) - timedelta(days=7)).isoformat()
client.create_shard_key(collection_name, today)
client.delete_shard_key(collection_name, oldest_shard_key)
```
@@ -0,0 +1,20 @@
```rust
let today = "2026-04-08";
let oldest_shard_key = (NaiveDate::parse_from_str(today, "%Y-%m-%d")?
- chrono::Duration::days(7))
.to_string();
client
.create_shard_key(
CreateShardKeyRequestBuilder::new(collection_name)
.request(CreateShardKeyBuilder::default().shard_key(today.to_string())),
)
.await?;
client
.delete_shard_key(
DeleteShardKeyRequestBuilder::new(collection_name)
.key(shard_key::Key::Keyword(oldest_shard_key)),
)
.await?;
```
@@ -0,0 +1,9 @@
```typescript
const today = "2026-04-08";
const oldestDate = new Date(today);
oldestDate.setDate(oldestDate.getDate() - 7);
const oldestShardKey = oldestDate.toISOString().slice(0, 10);
await client.createShardKey(collectionName, { shard_key: today });
await client.deleteShardKey(collectionName, { shard_key: oldestShardKey });
```
@@ -0,0 +1,138 @@
```python
from qdrant_client import QdrantClient, models
client = QdrantClient(
url=QDRANT_URL,
api_key=QDRANT_API_KEY,
cloud_inference=True
)
from qdrant_client import models
collection_name = "my_collection"
if client.collection_exists(collection_name=collection_name):
client.delete_collection(collection_name=collection_name)
client.create_collection(
collection_name=collection_name,
vectors_config={
"dense_vector": models.VectorParams(
size=384, distance=models.Distance.COSINE
)
},
sharding_method=models.ShardingMethod.CUSTOM
)
import csv
import urllib.request
def parse_csv(url):
with urllib.request.urlopen(url) as response:
reader = csv.DictReader(line.decode('utf-8') for line in response)
yield from reader
from qdrant_client.http.models import PointStruct, Document
import uuid
csv_url = 'https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv'
# Retrieve a list of existing shard keys in the collection
existing_shard_keys = list(client.list_shard_keys(collection_name=collection_name).shard_keys)
dense_model = "sentence-transformers/all-MiniLM-L6-v2"
batch_size = 100
current_date = None
buffer: list[PointStruct] = []
for row in parse_csv(csv_url):
shard_date = row['datetime'][:10] # Extract YYYY-MM-DD
if shard_date != current_date:
# Flush buffer for the previous date before switching
if buffer:
client.upload_points(
collection_name=collection_name,
points=buffer,
shard_key_selector=current_date,
)
buffer = []
# Create shard for the new date if it doesn't exist yet
if shard_date not in existing_shard_keys:
client.create_shard_key(collection_name, shard_date)
existing_shard_keys.append(shard_date)
current_date = shard_date
# Add point to buffer
buffer.append(PointStruct(
id=uuid.uuid4().hex,
payload={"text": row['text'], "datetime": row['datetime']},
vector={"dense_vector": Document(text=row["text"], model=dense_model)}
))
# Flush batch if buffer size exceeds batch size
if len(buffer) >= batch_size:
client.upload_points(
collection_name=collection_name,
points=buffer,
shard_key_selector=current_date,
)
buffer = []
# Flush remaining partial batch
if buffer:
client.upload_points(
collection_name=collection_name,
points=buffer,
shard_key_selector=current_date,
)
query_text = "coffee"
resp = client.query_points(
collection_name=collection_name,
query=Document(text=query_text, model=dense_model),
using="dense_vector",
limit=5,
shard_key_selector="2026-04-07"
)
print(resp)
resp = client.query_points(
collection_name=collection_name,
query=Document(text=query_text, model=dense_model),
using="dense_vector",
limit=5,
shard_key_selector=["2026-04-06","2026-04-07"]
)
print(resp)
resp = client.query_points(
collection_name=collection_name,
query=Document(text=query_text, model=dense_model),
using="dense_vector",
limit=5,
)
print(resp)
from datetime import date, timedelta
today = "2026-04-08"
oldest_shard_key = (date.fromisoformat(today) - timedelta(days=7)).isoformat()
client.create_shard_key(collection_name, today)
client.delete_shard_key(collection_name, oldest_shard_key)
client.upsert(
collection_name=collection_name,
points=[PointStruct(
id=uuid.uuid4().hex,
payload={"text": "The best way to start a Wednesday is with a cup of coffee", "datetime": "2026-04-08T07:57:47"},
vector={
"dense_vector": Document(text="The best way to start a Wednesday is with a cup of coffee", model=dense_model)
})],
shard_key_selector=today
)
```
@@ -0,0 +1,235 @@
```rust
use std::collections::{HashMap, HashSet};
use chrono::NaiveDate;
use qdrant_client::Qdrant;
use qdrant_client::qdrant::{
CreateCollectionBuilder, CreateShardKeyBuilder, CreateShardKeyRequestBuilder,
DeleteShardKeyRequestBuilder, Distance, Document, DocumentBuilder,
PointStruct, Query, QueryPointsBuilder, ShardKeySelector, ShardingMethod,
UpsertPointsBuilder, VectorParamsBuilder, VectorsConfigBuilder, shard_key,
};
let client = Qdrant::from_url(QDRANT_URL)
.api_key(QDRANT_API_KEY)
.build()?;
let collection_name = "my_collection";
if client.collection_exists(collection_name).await? {
client.delete_collection(collection_name).await?;
}
let mut vectors_config = VectorsConfigBuilder::default();
vectors_config.add_named_vector_params(
"dense_vector",
VectorParamsBuilder::new(384, Distance::Cosine),
);
client
.create_collection(
CreateCollectionBuilder::new(collection_name)
.vectors_config(vectors_config)
.sharding_method(ShardingMethod::Custom.into()),
)
.await?;
struct CsvRow {
text: String,
datetime: String,
}
fn parse_csv(url: &str) -> anyhow::Result<impl Iterator<Item = anyhow::Result<CsvRow>>> {
let reader = ureq::get(url).call()?.into_body().into_reader();
let mut rdr = csv::Reader::from_reader(reader);
let headers = rdr.headers()?.clone();
let text_idx = headers.iter().position(|h| h == "text").unwrap();
let datetime_idx = headers.iter().position(|h| h == "datetime").unwrap();
let iter = rdr.into_records().map(move |result| {
let record = result?;
Ok(CsvRow {
text: record[text_idx].to_string(),
datetime: record[datetime_idx].to_string(),
})
});
Ok(iter)
}
let csv_url = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv";
// Retrieve a list of existing shard keys in the collection
let response = client.list_shard_keys(collection_name).await?;
let mut existing_shard_keys: HashSet<String> = response
.shard_keys
.into_iter()
.filter_map(|d| {
d.key?.key.and_then(|k| match k {
shard_key::Key::Keyword(s) => Some(s),
_ => None,
})
})
.collect();
let dense_model = "sentence-transformers/all-MiniLM-L6-v2";
let batch_size = 100;
let mut current_date = String::new();
let mut buffer: Vec<PointStruct> = Vec::new();
for row in parse_csv(csv_url)? {
let row = row?;
let text = row.text;
let datetime = row.datetime;
let shard_date = datetime[..10].to_string(); // Extract YYYY-MM-DD
if shard_date != current_date {
// Flush buffer for the previous date before switching
if !buffer.is_empty() {
client
.upsert_points(
UpsertPointsBuilder::new(
collection_name,
std::mem::take(&mut buffer),
)
.shard_key_selector(current_date.clone()),
)
.await?;
}
// Create shard for the new date if it doesn't exist yet
if !existing_shard_keys.contains(&shard_date) {
client
.create_shard_key(
CreateShardKeyRequestBuilder::new(collection_name).request(
CreateShardKeyBuilder::default().shard_key(shard_date.clone()),
),
)
.await?;
existing_shard_keys.insert(shard_date.clone());
}
current_date = shard_date;
}
// Add point to buffer
buffer.push(PointStruct::new(
uuid::Uuid::new_v4().to_string(),
HashMap::from([(
"dense_vector".to_string(),
DocumentBuilder::new(&text, dense_model).build(),
)]),
[("text", text.into()), ("datetime", datetime.into())],
));
// Flush batch if buffer size exceeds batch size
if buffer.len() >= batch_size {
client
.upsert_points(
UpsertPointsBuilder::new(collection_name, std::mem::take(&mut buffer))
.shard_key_selector(current_date.clone()),
)
.await?;
}
}
// Flush remaining partial batch
if !buffer.is_empty() {
client
.upsert_points(
UpsertPointsBuilder::new(collection_name, buffer)
.shard_key_selector(current_date.clone()),
)
.await?;
}
let query_text = "coffee";
let result = client
.query(
QueryPointsBuilder::new(collection_name)
.query(Query::new_nearest(Document::new(query_text, dense_model)))
.using("dense_vector")
.limit(5)
.shard_key_selector("2026-04-07".to_string()),
)
.await?;
for hit in result.result {
println!("{:?}", hit);
}
let result = client
.query(
QueryPointsBuilder::new(collection_name)
.query(Query::new_nearest(Document::new(query_text, dense_model)))
.using("dense_vector")
.limit(5)
.shard_key_selector(ShardKeySelector {
shard_keys: vec![
"2026-04-06".to_string().into(),
"2026-04-07".to_string().into(),
],
fallback: None,
}),
)
.await?;
for hit in result.result {
println!("{:?}", hit);
}
let result = client
.query(
QueryPointsBuilder::new(collection_name)
.query(Query::new_nearest(Document::new(query_text, dense_model)))
.using("dense_vector")
.limit(5),
)
.await?;
for hit in result.result {
println!("{:?}", hit);
}
let today = "2026-04-08";
let oldest_shard_key = (NaiveDate::parse_from_str(today, "%Y-%m-%d")?
- chrono::Duration::days(7))
.to_string();
client
.create_shard_key(
CreateShardKeyRequestBuilder::new(collection_name)
.request(CreateShardKeyBuilder::default().shard_key(today.to_string())),
)
.await?;
client
.delete_shard_key(
DeleteShardKeyRequestBuilder::new(collection_name)
.key(shard_key::Key::Keyword(oldest_shard_key)),
)
.await?;
client
.upsert_points(
UpsertPointsBuilder::new(
collection_name,
vec![PointStruct::new(
uuid::Uuid::new_v4().to_string(),
HashMap::from([(
"dense_vector".to_string(),
DocumentBuilder::new(
"The best way to start a Wednesday is with a cup of coffee",
dense_model,
)
.build(),
)]),
[
("text", "The best way to start a Wednesday is with a cup of coffee".into()),
("datetime", "2026-04-08T07:57:47".into()),
],
)],
)
.shard_key_selector(today.to_string()),
)
.await?;
```
@@ -0,0 +1,11 @@
```csharp
result = await client.QueryAsync(
collectionName: collectionName,
query: new Document { Text = queryText, Model = denseModel },
usingVector: "dense_vector",
limit: 5
);
foreach (var hit in result)
Console.WriteLine(hit);
```
@@ -0,0 +1,12 @@
```go
result, err = client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: collectionName,
Query: qdrant.NewQueryDocument(&qdrant.Document{Text: queryText, Model: denseModel}),
Using: qdrant.PtrOf("dense_vector"),
Limit: qdrant.PtrOf(uint64(5)),
})
for _, hit := range result {
fmt.Println(hit)
}
```
@@ -0,0 +1,14 @@
```java
result = client.queryAsync(
QueryPoints.newBuilder()
.setCollectionName(collectionName)
.setQuery(nearest(Document.newBuilder().setText(queryText).setModel(denseModel).build()))
.setUsing("dense_vector")
.setLimit(5)
.build()
).get();
for (var hit : result) {
System.out.println(hit);
}
```
@@ -0,0 +1,9 @@
```python
resp = client.query_points(
collection_name=collection_name,
query=Document(text=query_text, model=dense_model),
using="dense_vector",
limit=5,
)
print(resp)
```
@@ -0,0 +1,14 @@
```rust
let result = client
.query(
QueryPointsBuilder::new(collection_name)
.query(Query::new_nearest(Document::new(query_text, dense_model)))
.using("dense_vector")
.limit(5),
)
.await?;
for hit in result.result {
println!("{:?}", hit);
}
```
@@ -0,0 +1,11 @@
```typescript
const allShardsResult = await client.query(collectionName, {
query: { text: queryText, model: denseModel },
using: "dense_vector",
limit: 5,
});
for (const hit of allShardsResult.points) {
console.log(hit);
}
```
@@ -0,0 +1,15 @@
```csharp
result = await client.QueryAsync(
collectionName: collectionName,
query: new Document { Text = queryText, Model = denseModel },
usingVector: "dense_vector",
limit: 5,
shardKeySelector: new ShardKeySelector
{
ShardKeys = { new List<ShardKey> { "2026-04-06", "2026-04-07" } }
}
);
foreach (var hit in result)
Console.WriteLine(hit);
```
@@ -0,0 +1,18 @@
```go
result, err = client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: collectionName,
Query: qdrant.NewQueryDocument(&qdrant.Document{Text: queryText, Model: denseModel}),
Using: qdrant.PtrOf("dense_vector"),
Limit: qdrant.PtrOf(uint64(5)),
ShardKeySelector: &qdrant.ShardKeySelector{
ShardKeys: []*qdrant.ShardKey{
qdrant.NewShardKey("2026-04-06"),
qdrant.NewShardKey("2026-04-07"),
},
},
})
for _, hit := range result {
fmt.Println(hit)
}
```
@@ -0,0 +1,18 @@
```java
result = client.queryAsync(
QueryPoints.newBuilder()
.setCollectionName(collectionName)
.setQuery(nearest(Document.newBuilder().setText(queryText).setModel(denseModel).build()))
.setUsing("dense_vector")
.setLimit(5)
.setShardKeySelector(ShardKeySelector.newBuilder()
.addShardKeys(shardKey("2026-04-06"))
.addShardKeys(shardKey("2026-04-07"))
.build())
.build()
).get();
for (var hit : result) {
System.out.println(hit);
}
```
@@ -0,0 +1,10 @@
```python
resp = client.query_points(
collection_name=collection_name,
query=Document(text=query_text, model=dense_model),
using="dense_vector",
limit=5,
shard_key_selector=["2026-04-06","2026-04-07"]
)
print(resp)
```
@@ -0,0 +1,21 @@
```rust
let result = client
.query(
QueryPointsBuilder::new(collection_name)
.query(Query::new_nearest(Document::new(query_text, dense_model)))
.using("dense_vector")
.limit(5)
.shard_key_selector(ShardKeySelector {
shard_keys: vec![
"2026-04-06".to_string().into(),
"2026-04-07".to_string().into(),
],
fallback: None,
}),
)
.await?;
for hit in result.result {
println!("{:?}", hit);
}
```
@@ -0,0 +1,12 @@
```typescript
const multiShardResult = await client.query(collectionName, {
query: { text: queryText, model: denseModel },
using: "dense_vector",
limit: 5,
shard_key: ["2026-04-06", "2026-04-07"],
});
for (const hit of multiShardResult.points) {
console.log(hit);
}
```
@@ -0,0 +1,17 @@
```csharp
string queryText = "coffee";
var result = await client.QueryAsync(
collectionName: collectionName,
query: new Document { Text = queryText, Model = denseModel },
usingVector: "dense_vector",
limit: 5,
shardKeySelector: new ShardKeySelector
{
ShardKeys = { new List<ShardKey> { "2026-04-07" } }
}
);
foreach (var hit in result)
Console.WriteLine(hit);
```
@@ -0,0 +1,17 @@
```go
queryText := "coffee"
result, err := client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: collectionName,
Query: qdrant.NewQueryDocument(&qdrant.Document{Text: queryText, Model: denseModel}),
Using: qdrant.PtrOf("dense_vector"),
Limit: qdrant.PtrOf(uint64(5)),
ShardKeySelector: &qdrant.ShardKeySelector{
ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey("2026-04-07")},
},
})
for _, hit := range result {
fmt.Println(hit)
}
```
@@ -0,0 +1,17 @@
```java
String queryText = "coffee";
var result = client.queryAsync(
QueryPoints.newBuilder()
.setCollectionName(collectionName)
.setQuery(nearest(Document.newBuilder().setText(queryText).setModel(denseModel).build()))
.setUsing("dense_vector")
.setLimit(5)
.setShardKeySelector(shardKeySelector("2026-04-07"))
.build()
).get();
for (var hit : result) {
System.out.println(hit);
}
```
@@ -0,0 +1,12 @@
```python
query_text = "coffee"
resp = client.query_points(
collection_name=collection_name,
query=Document(text=query_text, model=dense_model),
using="dense_vector",
limit=5,
shard_key_selector="2026-04-07"
)
print(resp)
```
@@ -0,0 +1,17 @@
```rust
let query_text = "coffee";
let result = client
.query(
QueryPointsBuilder::new(collection_name)
.query(Query::new_nearest(Document::new(query_text, dense_model)))
.using("dense_vector")
.limit(5)
.shard_key_selector("2026-04-07".to_string()),
)
.await?;
for hit in result.result {
println!("{:?}", hit);
}
```
@@ -0,0 +1,14 @@
```typescript
const queryText = "coffee";
const singleShardResult = await client.query(collectionName, {
query: { text: queryText, model: denseModel },
using: "dense_vector",
limit: 5,
shard_key: "2026-04-07",
});
for (const hit of singleShardResult.points) {
console.log(hit);
}
```
@@ -0,0 +1,189 @@
```typescript
import { QdrantClient } from "@qdrant/js-client-rest";
const client = new QdrantClient({
url: QDRANT_URL,
apiKey: QDRANT_API_KEY,
});
const collectionName = "my_collection";
if (await client.collectionExists(collectionName)) {
await client.deleteCollection(collectionName);
}
await client.createCollection(collectionName, {
vectors: {
dense_vector: {
size: 384,
distance: "Cosine",
},
},
sharding_method: "custom",
});
function parseCsvLine(line: string): string[] {
const fields: string[] = [];
let i = 0;
while (i < line.length) {
if (line[i] === '"') {
i++;
let field = "";
while (i < line.length) {
if (line[i] === '"' && line[i + 1] === '"') { field += '"'; i += 2; }
else if (line[i] === '"') { i++; break; }
else { field += line[i++]; }
}
fields.push(field);
if (line[i] === ",") i++;
} else {
const start = i;
while (i < line.length && line[i] !== ",") i++;
fields.push(line.slice(start, i));
if (i < line.length) i++;
}
}
return fields;
}
async function* parseCSV(url: string): AsyncGenerator<{ text: string; datetime: string }> {
const response = await fetch(url);
const reader = response.body!.getReader();
const decoder = new TextDecoder();
let remainder = "";
let headers: string[] | null = null;
let textIdx = -1;
let datetimeIdx = -1;
while (true) {
const { done, value } = await reader.read();
const chunk = done ? "" : decoder.decode(value, { stream: true });
const lines = (remainder + chunk).split("\n");
remainder = done ? "" : lines.pop()!;
for (const line of lines) {
if (!line.trim()) continue;
if (headers === null) {
headers = line.split(",");
textIdx = headers.indexOf("text");
datetimeIdx = headers.indexOf("datetime");
continue;
}
const fields = parseCsvLine(line);
yield { text: fields[textIdx], datetime: fields[datetimeIdx] };
}
if (done) break;
}
}
const csvUrl = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv";
// Retrieve a list of existing shard keys in the collection
const shardKeysResult = await client.listShardKeys(collectionName);
const existingShardKeys = new Set((shardKeysResult.shard_keys ?? []).map((d) => String(d.key)));
const denseModel = "sentence-transformers/all-MiniLM-L6-v2";
const batchSize = 100;
let currentDate = "";
let buffer: Extract<Parameters<typeof client.upsert>[1], { points: unknown }>['points'] = [];
for await (const { text: postText, datetime } of parseCSV(csvUrl)) {
const shardDate = datetime.slice(0, 10); // Extract YYYY-MM-DD
if (shardDate !== currentDate) {
// Flush buffer for the previous date before switching
if (buffer.length > 0) {
await client.upsert(collectionName, { points: buffer, shard_key: currentDate });
buffer = [];
}
// Create shard for the new date if it doesn't exist yet
if (!existingShardKeys.has(shardDate)) {
await client.createShardKey(collectionName, { shard_key: shardDate });
existingShardKeys.add(shardDate);
}
currentDate = shardDate;
}
// Add point to buffer
buffer.push({
id: crypto.randomUUID(),
vector: { dense_vector: { text: postText, model: denseModel } },
payload: { text: postText, datetime },
});
// Flush batch if buffer size exceeds batch size
if (buffer.length >= batchSize) {
await client.upsert(collectionName, { points: buffer, shard_key: currentDate });
buffer = [];
}
}
// Flush remaining partial batch
if (buffer.length > 0) {
await client.upsert(collectionName, { points: buffer, shard_key: currentDate });
}
const queryText = "coffee";
const singleShardResult = await client.query(collectionName, {
query: { text: queryText, model: denseModel },
using: "dense_vector",
limit: 5,
shard_key: "2026-04-07",
});
for (const hit of singleShardResult.points) {
console.log(hit);
}
const multiShardResult = await client.query(collectionName, {
query: { text: queryText, model: denseModel },
using: "dense_vector",
limit: 5,
shard_key: ["2026-04-06", "2026-04-07"],
});
for (const hit of multiShardResult.points) {
console.log(hit);
}
const allShardsResult = await client.query(collectionName, {
query: { text: queryText, model: denseModel },
using: "dense_vector",
limit: 5,
});
for (const hit of allShardsResult.points) {
console.log(hit);
}
const today = "2026-04-08";
const oldestDate = new Date(today);
oldestDate.setDate(oldestDate.getDate() - 7);
const oldestShardKey = oldestDate.toISOString().slice(0, 10);
await client.createShardKey(collectionName, { shard_key: today });
await client.deleteShardKey(collectionName, { shard_key: oldestShardKey });
await client.upsert(collectionName, {
points: [
{
id: crypto.randomUUID(),
vector: {
dense_vector: {
text: "The best way to start a Wednesday is with a cup of coffee",
model: denseModel,
},
},
payload: {
text: "The best way to start a Wednesday is with a cup of coffee",
datetime: "2026-04-08T07:57:47",
},
},
],
shard_key: today,
});
```
@@ -0,0 +1,85 @@
```csharp
string csvUrl = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv";
// Retrieve a list of existing shard keys in the collection
var existingShardKeys = (await client.ListShardKeysAsync(collectionName))
.Select(sk => sk.Key.Keyword)
.ToHashSet();
string denseModel = "sentence-transformers/all-MiniLM-L6-v2";
int batchSize = 100;
string? currentDate = null;
var buffer = new List<PointStruct>();
await foreach (var (text, datetime) in ParseCsv(csvUrl))
{
string shardDate = datetime[..10]; // Extract YYYY-MM-DD
if (shardDate != currentDate)
{
// Flush buffer for the previous date before switching
if (buffer.Count > 0)
{
await client.UpsertAsync(
collectionName: collectionName,
points: buffer,
shardKeySelector: new ShardKeySelector
{
ShardKeys = { new List<ShardKey> { currentDate! } }
}
);
buffer.Clear();
}
// Create shard for the new date if it doesn't exist yet
if (!existingShardKeys.Contains(shardDate))
{
await client.CreateShardKeyAsync(
collectionName,
new CreateShardKey { ShardKey = new ShardKey { Keyword = shardDate } }
);
existingShardKeys.Add(shardDate);
}
currentDate = shardDate;
}
// Add point to buffer
buffer.Add(new PointStruct
{
Id = Guid.NewGuid(),
Vectors = new Dictionary<string, Vector>
{
["dense_vector"] = new Document { Text = text, Model = denseModel }
},
Payload = { ["text"] = text, ["datetime"] = datetime }
});
// Flush batch if buffer size exceeds batch size
if (buffer.Count >= batchSize)
{
await client.UpsertAsync(
collectionName: collectionName,
points: buffer,
shardKeySelector: new ShardKeySelector
{
ShardKeys = { new List<ShardKey> { currentDate! } }
}
);
buffer.Clear();
}
}
// Flush remaining partial batch
if (buffer.Count > 0)
{
await client.UpsertAsync(
collectionName: collectionName,
points: buffer,
shardKeySelector: new ShardKeySelector
{
ShardKeys = { new List<ShardKey> { currentDate! } }
}
);
}
```
@@ -0,0 +1,84 @@
```go
csvUrl := "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv"
shardKeyDescriptions, err := client.ListShardKeys(context.Background(), collectionName)
// Retrieve a list of existing shard keys in the collection
existingShardKeys := make(map[string]bool)
for _, desc := range shardKeyDescriptions {
existingShardKeys[desc.Key.GetKeyword()] = true
}
denseModel := "sentence-transformers/all-MiniLM-L6-v2"
batchSize := 100
var currentDate string
var buffer []*qdrant.PointStruct
err = parseCSV(csvUrl, func(row CSVRow) {
text := row.Text
datetime := row.Datetime
shardDate := datetime[:10] // Extract YYYY-MM-DD
if shardDate != currentDate {
// Flush buffer for the previous date before switching
if len(buffer) > 0 {
client.Upsert(context.Background(), &qdrant.UpsertPoints{
CollectionName: collectionName,
Points: buffer,
ShardKeySelector: &qdrant.ShardKeySelector{
ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey(currentDate)},
},
})
buffer = nil
}
// Create shard for the new date if it doesn't exist yet
if !existingShardKeys[shardDate] {
client.CreateShardKey(context.Background(), collectionName, &qdrant.CreateShardKey{
ShardKey: qdrant.NewShardKey(shardDate),
})
existingShardKeys[shardDate] = true
}
currentDate = shardDate
}
// Add point to buffer
buffer = append(buffer, &qdrant.PointStruct{
Id: qdrant.NewID(uuid.New().String()),
Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{
"dense_vector": qdrant.NewVectorDocument(&qdrant.Document{
Text: text,
Model: denseModel,
}),
}),
Payload: qdrant.NewValueMap(map[string]any{
"text": text,
"datetime": datetime,
}),
})
// Flush batch if buffer size exceeds batch size
if len(buffer) >= batchSize {
client.Upsert(context.Background(), &qdrant.UpsertPoints{
CollectionName: collectionName,
Points: buffer,
ShardKeySelector: &qdrant.ShardKeySelector{
ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey(currentDate)},
},
})
buffer = nil
}
})
// Flush remaining partial batch
if len(buffer) > 0 {
client.Upsert(context.Background(), &qdrant.UpsertPoints{
CollectionName: collectionName,
Points: buffer,
ShardKeySelector: &qdrant.ShardKeySelector{
ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey(currentDate)},
},
})
}
```
@@ -0,0 +1,88 @@
```java
String csvUrl = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv";
// Retrieve a list of existing shard keys in the collection
var shardKeyDescriptions = client.listShardKeysAsync(collectionName).get();
Set<String> existingShardKeys = new HashSet<>();
for (var desc : shardKeyDescriptions) {
existingShardKeys.add(desc.getKey().getKeyword());
}
String denseModel = "sentence-transformers/all-MiniLM-L6-v2";
int batchSize = 100;
String currentDate = null;
List<PointStruct> buffer = new ArrayList<>();
try (var stream = parseCSV(csvUrl)) {
for (var row : (Iterable<CsvRow>) stream::iterator) {
String text = row.text;
String datetime = row.datetime;
String shardDate = datetime.substring(0, 10); // Extract YYYY-MM-DD
if (!shardDate.equals(currentDate)) {
// Flush buffer for the previous date before switching
if (!buffer.isEmpty()) {
client.upsertAsync(
UpsertPoints.newBuilder()
.setCollectionName(collectionName)
.addAllPoints(buffer)
.setShardKeySelector(shardKeySelector(currentDate))
.build()
).get();
buffer.clear();
}
// Create shard for the new date if it doesn't exist yet
if (!existingShardKeys.contains(shardDate)) {
client.createShardKeyAsync(
CreateShardKeyRequest.newBuilder()
.setCollectionName(collectionName)
.setRequest(CreateShardKey.newBuilder()
.setShardKey(shardKey(shardDate))
.build())
.build()
).get();
existingShardKeys.add(shardDate);
}
currentDate = shardDate;
}
// Add point to buffer
buffer.add(
PointStruct.newBuilder()
.setId(id(UUID.randomUUID()))
.setVectors(namedVectors(Map.of(
"dense_vector",
vector(Document.newBuilder()
.setText(text)
.setModel(denseModel)
.build()))))
.putAllPayload(Map.of("text", value(text), "datetime", value(datetime)))
.build());
// Flush batch if buffer size exceeds batch size
if (buffer.size() >= batchSize) {
client.upsertAsync(
UpsertPoints.newBuilder()
.setCollectionName(collectionName)
.addAllPoints(buffer)
.setShardKeySelector(shardKeySelector(currentDate))
.build()
).get();
buffer.clear();
}
}
}
// Flush remaining partial batch
if (!buffer.isEmpty()) {
client.upsertAsync(
UpsertPoints.newBuilder()
.setCollectionName(collectionName)
.addAllPoints(buffer)
.setShardKeySelector(shardKeySelector(currentDate))
.build()
).get();
}
```
@@ -0,0 +1,58 @@
```python
from qdrant_client.http.models import PointStruct, Document
import uuid
csv_url = 'https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv'
# Retrieve a list of existing shard keys in the collection
existing_shard_keys = list(client.list_shard_keys(collection_name=collection_name).shard_keys)
dense_model = "sentence-transformers/all-MiniLM-L6-v2"
batch_size = 100
current_date = None
buffer: list[PointStruct] = []
for row in parse_csv(csv_url):
shard_date = row['datetime'][:10] # Extract YYYY-MM-DD
if shard_date != current_date:
# Flush buffer for the previous date before switching
if buffer:
client.upload_points(
collection_name=collection_name,
points=buffer,
shard_key_selector=current_date,
)
buffer = []
# Create shard for the new date if it doesn't exist yet
if shard_date not in existing_shard_keys:
client.create_shard_key(collection_name, shard_date)
existing_shard_keys.append(shard_date)
current_date = shard_date
# Add point to buffer
buffer.append(PointStruct(
id=uuid.uuid4().hex,
payload={"text": row['text'], "datetime": row['datetime']},
vector={"dense_vector": Document(text=row["text"], model=dense_model)}
))
# Flush batch if buffer size exceeds batch size
if len(buffer) >= batch_size:
client.upload_points(
collection_name=collection_name,
points=buffer,
shard_key_selector=current_date,
)
buffer = []
# Flush remaining partial batch
if buffer:
client.upload_points(
collection_name=collection_name,
points=buffer,
shard_key_selector=current_date,
)
```
@@ -0,0 +1,87 @@
```rust
let csv_url = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv";
// Retrieve a list of existing shard keys in the collection
let response = client.list_shard_keys(collection_name).await?;
let mut existing_shard_keys: HashSet<String> = response
.shard_keys
.into_iter()
.filter_map(|d| {
d.key?.key.and_then(|k| match k {
shard_key::Key::Keyword(s) => Some(s),
_ => None,
})
})
.collect();
let dense_model = "sentence-transformers/all-MiniLM-L6-v2";
let batch_size = 100;
let mut current_date = String::new();
let mut buffer: Vec<PointStruct> = Vec::new();
for row in parse_csv(csv_url)? {
let row = row?;
let text = row.text;
let datetime = row.datetime;
let shard_date = datetime[..10].to_string(); // Extract YYYY-MM-DD
if shard_date != current_date {
// Flush buffer for the previous date before switching
if !buffer.is_empty() {
client
.upsert_points(
UpsertPointsBuilder::new(
collection_name,
std::mem::take(&mut buffer),
)
.shard_key_selector(current_date.clone()),
)
.await?;
}
// Create shard for the new date if it doesn't exist yet
if !existing_shard_keys.contains(&shard_date) {
client
.create_shard_key(
CreateShardKeyRequestBuilder::new(collection_name).request(
CreateShardKeyBuilder::default().shard_key(shard_date.clone()),
),
)
.await?;
existing_shard_keys.insert(shard_date.clone());
}
current_date = shard_date;
}
// Add point to buffer
buffer.push(PointStruct::new(
uuid::Uuid::new_v4().to_string(),
HashMap::from([(
"dense_vector".to_string(),
DocumentBuilder::new(&text, dense_model).build(),
)]),
[("text", text.into()), ("datetime", datetime.into())],
));
// Flush batch if buffer size exceeds batch size
if buffer.len() >= batch_size {
client
.upsert_points(
UpsertPointsBuilder::new(collection_name, std::mem::take(&mut buffer))
.shard_key_selector(current_date.clone()),
)
.await?;
}
}
// Flush remaining partial batch
if !buffer.is_empty() {
client
.upsert_points(
UpsertPointsBuilder::new(collection_name, buffer)
.shard_key_selector(current_date.clone()),
)
.await?;
}
```
@@ -0,0 +1,50 @@
```typescript
const csvUrl = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv";
// Retrieve a list of existing shard keys in the collection
const shardKeysResult = await client.listShardKeys(collectionName);
const existingShardKeys = new Set((shardKeysResult.shard_keys ?? []).map((d) => String(d.key)));
const denseModel = "sentence-transformers/all-MiniLM-L6-v2";
const batchSize = 100;
let currentDate = "";
let buffer: Extract<Parameters<typeof client.upsert>[1], { points: unknown }>['points'] = [];
for await (const { text: postText, datetime } of parseCSV(csvUrl)) {
const shardDate = datetime.slice(0, 10); // Extract YYYY-MM-DD
if (shardDate !== currentDate) {
// Flush buffer for the previous date before switching
if (buffer.length > 0) {
await client.upsert(collectionName, { points: buffer, shard_key: currentDate });
buffer = [];
}
// Create shard for the new date if it doesn't exist yet
if (!existingShardKeys.has(shardDate)) {
await client.createShardKey(collectionName, { shard_key: shardDate });
existingShardKeys.add(shardDate);
}
currentDate = shardDate;
}
// Add point to buffer
buffer.push({
id: crypto.randomUUID(),
vector: { dense_vector: { text: postText, model: denseModel } },
payload: { text: postText, datetime },
});
// Flush batch if buffer size exceeds batch size
if (buffer.length >= batchSize) {
await client.upsert(collectionName, { points: buffer, shard_key: currentDate });
buffer = [];
}
}
// Flush remaining partial batch
if (buffer.length > 0) {
await client.upsert(collectionName, { points: buffer, shard_key: currentDate });
}
```
@@ -0,0 +1,292 @@
package snippet
import (
"context"
"encoding/csv"
"fmt"
"io"
"net/http"
"time"
"github.com/google/uuid"
"github.com/qdrant/go-client/qdrant"
)
// @block-start parse-csv
type CSVRow struct {
Text string
Datetime string
}
func parseCSV(url string, fn func(CSVRow)) error {
resp, err := http.Get(url)
if err != nil {
return err
}
defer resp.Body.Close()
csvReader := csv.NewReader(resp.Body)
headers, err := csvReader.Read()
if err != nil {
return err
}
textIdx, datetimeIdx := -1, -1
for i, h := range headers {
switch h {
case "text":
textIdx = i
case "datetime":
datetimeIdx = i
}
}
for {
row, err := csvReader.Read()
if err == io.EOF {
break
}
if err != nil {
return err
}
fn(CSVRow{Text: row[textIdx], Datetime: row[datetimeIdx]})
}
return nil
}
// @block-end parse-csv
func Main() {
// @hide-start
QDRANT_URL := ""
QDRANT_API_KEY := ""
// @hide-end
// @block-start initialize-client
client, err := qdrant.NewClient(&qdrant.Config{
Host: QDRANT_URL,
APIKey: QDRANT_API_KEY,
UseTLS: true,
})
// @block-end initialize-client
// @hide-start
if err != nil {
panic(err)
}
// @hide-end
// @block-start create-collection
collectionName := "my_collection"
exists, err := client.CollectionExists(context.Background(), collectionName)
if err != nil { panic(err) } // @hide
if exists {
client.DeleteCollection(context.Background(), collectionName)
}
client.CreateCollection(context.Background(), &qdrant.CreateCollection{
CollectionName: collectionName,
VectorsConfig: qdrant.NewVectorsConfigMap(
map[string]*qdrant.VectorParams{
"dense_vector": {
Size: 384,
Distance: qdrant.Distance_Cosine,
},
},
),
ShardingMethod: qdrant.ShardingMethod_Custom.Enum(),
})
// @block-end create-collection
// @block-start upload-vectors
csvUrl := "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv"
shardKeyDescriptions, err := client.ListShardKeys(context.Background(), collectionName)
if err != nil { panic(err) } // @hide
// Retrieve a list of existing shard keys in the collection
existingShardKeys := make(map[string]bool)
for _, desc := range shardKeyDescriptions {
existingShardKeys[desc.Key.GetKeyword()] = true
}
denseModel := "sentence-transformers/all-MiniLM-L6-v2"
batchSize := 100
var currentDate string
var buffer []*qdrant.PointStruct
err = parseCSV(csvUrl, func(row CSVRow) {
text := row.Text
datetime := row.Datetime
shardDate := datetime[:10] // Extract YYYY-MM-DD
if shardDate != currentDate {
// Flush buffer for the previous date before switching
if len(buffer) > 0 {
client.Upsert(context.Background(), &qdrant.UpsertPoints{
CollectionName: collectionName,
Points: buffer,
ShardKeySelector: &qdrant.ShardKeySelector{
ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey(currentDate)},
},
})
buffer = nil
}
// Create shard for the new date if it doesn't exist yet
if !existingShardKeys[shardDate] {
client.CreateShardKey(context.Background(), collectionName, &qdrant.CreateShardKey{
ShardKey: qdrant.NewShardKey(shardDate),
})
existingShardKeys[shardDate] = true
}
currentDate = shardDate
}
// Add point to buffer
buffer = append(buffer, &qdrant.PointStruct{
Id: qdrant.NewID(uuid.New().String()),
Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{
"dense_vector": qdrant.NewVectorDocument(&qdrant.Document{
Text: text,
Model: denseModel,
}),
}),
Payload: qdrant.NewValueMap(map[string]any{
"text": text,
"datetime": datetime,
}),
})
// Flush batch if buffer size exceeds batch size
if len(buffer) >= batchSize {
client.Upsert(context.Background(), &qdrant.UpsertPoints{
CollectionName: collectionName,
Points: buffer,
ShardKeySelector: &qdrant.ShardKeySelector{
ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey(currentDate)},
},
})
buffer = nil
}
})
if err != nil { panic(err) } // @hide
// Flush remaining partial batch
if len(buffer) > 0 {
client.Upsert(context.Background(), &qdrant.UpsertPoints{
CollectionName: collectionName,
Points: buffer,
ShardKeySelector: &qdrant.ShardKeySelector{
ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey(currentDate)},
},
})
}
// @block-end upload-vectors
// @block-start search-single-shard
queryText := "coffee"
result, err := client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: collectionName,
Query: qdrant.NewQueryDocument(&qdrant.Document{Text: queryText, Model: denseModel}),
Using: qdrant.PtrOf("dense_vector"),
Limit: qdrant.PtrOf(uint64(5)),
ShardKeySelector: &qdrant.ShardKeySelector{
ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey("2026-04-07")},
},
})
// @hide-start
if err != nil {
panic(err)
}
// @hide-end
for _, hit := range result {
fmt.Println(hit)
}
// @block-end search-single-shard
// @block-start search-multiple-shards
result, err = client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: collectionName,
Query: qdrant.NewQueryDocument(&qdrant.Document{Text: queryText, Model: denseModel}),
Using: qdrant.PtrOf("dense_vector"),
Limit: qdrant.PtrOf(uint64(5)),
ShardKeySelector: &qdrant.ShardKeySelector{
ShardKeys: []*qdrant.ShardKey{
qdrant.NewShardKey("2026-04-06"),
qdrant.NewShardKey("2026-04-07"),
},
},
})
// @hide-start
if err != nil {
panic(err)
}
// @hide-end
for _, hit := range result {
fmt.Println(hit)
}
// @block-end search-multiple-shards
// @block-start search-all-shards
result, err = client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: collectionName,
Query: qdrant.NewQueryDocument(&qdrant.Document{Text: queryText, Model: denseModel}),
Using: qdrant.PtrOf("dense_vector"),
Limit: qdrant.PtrOf(uint64(5)),
})
// @hide-start
if err != nil {
panic(err)
}
// @hide-end
for _, hit := range result {
fmt.Println(hit)
}
// @block-end search-all-shards
// @block-start pruning-shards
today := "2026-04-08"
t, _ := time.Parse("2006-01-02", today)
oldestShardKey := t.AddDate(0, 0, -7).Format("2006-01-02")
client.CreateShardKey(context.Background(), collectionName, &qdrant.CreateShardKey{
ShardKey: qdrant.NewShardKey(today),
})
client.DeleteShardKey(context.Background(), collectionName, &qdrant.DeleteShardKey{
ShardKey: qdrant.NewShardKey(oldestShardKey),
})
// @block-end pruning-shards
// @block-start ingest-new-data
client.Upsert(context.Background(), &qdrant.UpsertPoints{
CollectionName: collectionName,
Points: []*qdrant.PointStruct{
{
Id: qdrant.NewID(uuid.New().String()),
Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{
"dense_vector": qdrant.NewVectorDocument(&qdrant.Document{
Text: "The best way to start a Wednesday is with a cup of coffee",
Model: denseModel,
}),
}),
Payload: qdrant.NewValueMap(map[string]any{
"text": "The best way to start a Wednesday is with a cup of coffee",
"datetime": "2026-04-08T07:57:47",
}),
},
},
ShardKeySelector: &qdrant.ShardKeySelector{
ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey(today)},
},
})
// @block-end ingest-new-data
}
@@ -0,0 +1,306 @@
package com.example.snippets_amalgamation;
import static io.qdrant.client.PointIdFactory.id;
import static io.qdrant.client.QueryFactory.nearest;
import static io.qdrant.client.ShardKeyFactory.shardKey;
import static io.qdrant.client.ShardKeySelectorFactory.shardKeySelector;
import static io.qdrant.client.ValueFactory.value;
import static io.qdrant.client.VectorFactory.vector;
import static io.qdrant.client.VectorsFactory.namedVectors;
import io.qdrant.client.QdrantClient;
import io.qdrant.client.QdrantGrpcClient;
import io.qdrant.client.grpc.Collections.CreateCollection;
import io.qdrant.client.grpc.Collections.CreateShardKey;
import io.qdrant.client.grpc.Collections.CreateShardKeyRequest;
import io.qdrant.client.grpc.Collections.DeleteShardKey;
import io.qdrant.client.grpc.Collections.DeleteShardKeyRequest;
import io.qdrant.client.grpc.Collections.Distance;
import io.qdrant.client.grpc.Collections.ShardingMethod;
import io.qdrant.client.grpc.Collections.VectorParams;
import io.qdrant.client.grpc.Collections.VectorParamsMap;
import io.qdrant.client.grpc.Collections.VectorsConfig;
import io.qdrant.client.grpc.Points.Document;
import io.qdrant.client.grpc.Points.PointStruct;
import io.qdrant.client.grpc.Points.QueryPoints;
import io.qdrant.client.grpc.Points.ShardKeySelector;
import io.qdrant.client.grpc.Points.UpsertPoints;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.URL;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.function.Function;
import java.util.stream.Stream;
public class Snippet {
// @block-start parse-csv
static class CsvRow {
final String text;
final String datetime;
CsvRow(String text, String datetime) { this.text = text; this.datetime = datetime; }
}
static Stream<CsvRow> parseCSV(String url) throws Exception {
Function<String, List<String>> parseCsvLine = line -> {
List<String> fields = new ArrayList<>();
boolean inQuotes = false;
var sb = new StringBuilder();
for (char c : line.toCharArray()) {
if (c == '"') {
inQuotes = !inQuotes;
} else if (c == ',' && !inQuotes) {
fields.add(sb.toString());
sb.setLength(0);
} else {
sb.append(c);
}
}
fields.add(sb.toString());
return fields;
};
var reader = new BufferedReader(new InputStreamReader(new URL(url).openStream()));
String headerLine = reader.readLine();
List<String> headers = List.of(headerLine.split(","));
int textIdx = headers.indexOf("text");
int datetimeIdx = headers.indexOf("datetime");
return reader.lines()
.map(line -> {
List<String> fields = parseCsvLine.apply(line);
return new CsvRow(fields.get(textIdx), fields.get(datetimeIdx));
})
.onClose(() -> { try { reader.close(); } catch (Exception ignored) {} });
}
// @block-end parse-csv
public static void run() throws Exception {
// @hide-start
String QDRANT_URL = "";
String QDRANT_API_KEY = "";
// @hide-end
// @block-start initialize-client
QdrantClient client =
new QdrantClient(
QdrantGrpcClient.newBuilder(QDRANT_URL, 6334, true)
.withApiKey(QDRANT_API_KEY)
.build());
// @block-end initialize-client
// @block-start create-collection
String collectionName = "my_collection";
if (client.collectionExistsAsync(collectionName).get()) {
client.deleteCollectionAsync(collectionName).get();
}
client.createCollectionAsync(
CreateCollection.newBuilder()
.setCollectionName(collectionName)
.setVectorsConfig(VectorsConfig.newBuilder().setParamsMap(
VectorParamsMap.newBuilder().putAllMap(Map.of(
"dense_vector",
VectorParams.newBuilder()
.setSize(384)
.setDistance(Distance.Cosine)
.build()))))
.setShardingMethod(ShardingMethod.Custom)
.build()
).get();
// @block-end create-collection
// @block-start upload-vectors
String csvUrl = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv";
// Retrieve a list of existing shard keys in the collection
var shardKeyDescriptions = client.listShardKeysAsync(collectionName).get();
Set<String> existingShardKeys = new HashSet<>();
for (var desc : shardKeyDescriptions) {
existingShardKeys.add(desc.getKey().getKeyword());
}
String denseModel = "sentence-transformers/all-MiniLM-L6-v2";
int batchSize = 100;
String currentDate = null;
List<PointStruct> buffer = new ArrayList<>();
try (var stream = parseCSV(csvUrl)) {
for (var row : (Iterable<CsvRow>) stream::iterator) {
String text = row.text;
String datetime = row.datetime;
String shardDate = datetime.substring(0, 10); // Extract YYYY-MM-DD
if (!shardDate.equals(currentDate)) {
// Flush buffer for the previous date before switching
if (!buffer.isEmpty()) {
client.upsertAsync(
UpsertPoints.newBuilder()
.setCollectionName(collectionName)
.addAllPoints(buffer)
.setShardKeySelector(shardKeySelector(currentDate))
.build()
).get();
buffer.clear();
}
// Create shard for the new date if it doesn't exist yet
if (!existingShardKeys.contains(shardDate)) {
client.createShardKeyAsync(
CreateShardKeyRequest.newBuilder()
.setCollectionName(collectionName)
.setRequest(CreateShardKey.newBuilder()
.setShardKey(shardKey(shardDate))
.build())
.build()
).get();
existingShardKeys.add(shardDate);
}
currentDate = shardDate;
}
// Add point to buffer
buffer.add(
PointStruct.newBuilder()
.setId(id(UUID.randomUUID()))
.setVectors(namedVectors(Map.of(
"dense_vector",
vector(Document.newBuilder()
.setText(text)
.setModel(denseModel)
.build()))))
.putAllPayload(Map.of("text", value(text), "datetime", value(datetime)))
.build());
// Flush batch if buffer size exceeds batch size
if (buffer.size() >= batchSize) {
client.upsertAsync(
UpsertPoints.newBuilder()
.setCollectionName(collectionName)
.addAllPoints(buffer)
.setShardKeySelector(shardKeySelector(currentDate))
.build()
).get();
buffer.clear();
}
}
}
// Flush remaining partial batch
if (!buffer.isEmpty()) {
client.upsertAsync(
UpsertPoints.newBuilder()
.setCollectionName(collectionName)
.addAllPoints(buffer)
.setShardKeySelector(shardKeySelector(currentDate))
.build()
).get();
}
// @block-end upload-vectors
// @block-start search-single-shard
String queryText = "coffee";
var result = client.queryAsync(
QueryPoints.newBuilder()
.setCollectionName(collectionName)
.setQuery(nearest(Document.newBuilder().setText(queryText).setModel(denseModel).build()))
.setUsing("dense_vector")
.setLimit(5)
.setShardKeySelector(shardKeySelector("2026-04-07"))
.build()
).get();
for (var hit : result) {
System.out.println(hit);
}
// @block-end search-single-shard
// @block-start search-multiple-shards
result = client.queryAsync(
QueryPoints.newBuilder()
.setCollectionName(collectionName)
.setQuery(nearest(Document.newBuilder().setText(queryText).setModel(denseModel).build()))
.setUsing("dense_vector")
.setLimit(5)
.setShardKeySelector(ShardKeySelector.newBuilder()
.addShardKeys(shardKey("2026-04-06"))
.addShardKeys(shardKey("2026-04-07"))
.build())
.build()
).get();
for (var hit : result) {
System.out.println(hit);
}
// @block-end search-multiple-shards
// @block-start search-all-shards
result = client.queryAsync(
QueryPoints.newBuilder()
.setCollectionName(collectionName)
.setQuery(nearest(Document.newBuilder().setText(queryText).setModel(denseModel).build()))
.setUsing("dense_vector")
.setLimit(5)
.build()
).get();
for (var hit : result) {
System.out.println(hit);
}
// @block-end search-all-shards
// @block-start pruning-shards
String today = "2026-04-08";
String oldestShardKey = LocalDate.parse(today).minusDays(7).toString();
client.createShardKeyAsync(
CreateShardKeyRequest.newBuilder()
.setCollectionName(collectionName)
.setRequest(CreateShardKey.newBuilder()
.setShardKey(shardKey(today))
.build())
.build()
).get();
client.deleteShardKeyAsync(
DeleteShardKeyRequest.newBuilder()
.setCollectionName(collectionName)
.setRequest(DeleteShardKey.newBuilder()
.setShardKey(shardKey(oldestShardKey))
.build())
.build()
).get();
// @block-end pruning-shards
// @block-start ingest-new-data
client.upsertAsync(
UpsertPoints.newBuilder()
.setCollectionName(collectionName)
.addAllPoints(List.of(
PointStruct.newBuilder()
.setId(id(UUID.randomUUID()))
.setVectors(namedVectors(Map.of(
"dense_vector",
vector(Document.newBuilder()
.setText("The best way to start a Wednesday is with a cup of coffee")
.setModel(denseModel)
.build()))))
.putAllPayload(Map.of(
"text", value("The best way to start a Wednesday is with a cup of coffee"),
"datetime", value("2026-04-08T07:57:47")))
.build()))
.setShardKeySelector(shardKeySelector(today))
.build()
).get();
// @block-end ingest-new-data
}
}
@@ -0,0 +1,160 @@
# @hide-start
# mypy: disable-error-code="arg-type"
QDRANT_URL=""
QDRANT_API_KEY=""
# @hide-end
# @block-start initialize-client
from qdrant_client import QdrantClient, models
client = QdrantClient(
url=QDRANT_URL,
api_key=QDRANT_API_KEY,
cloud_inference=True
)
# @block-end initialize-client
# @block-start create-collection
from qdrant_client import models
collection_name = "my_collection"
if client.collection_exists(collection_name=collection_name):
client.delete_collection(collection_name=collection_name)
client.create_collection(
collection_name=collection_name,
vectors_config={
"dense_vector": models.VectorParams(
size=384, distance=models.Distance.COSINE
)
},
sharding_method=models.ShardingMethod.CUSTOM
)
# @block-end create-collection
# @block-start parse-csv
import csv
import urllib.request
def parse_csv(url):
with urllib.request.urlopen(url) as response:
reader = csv.DictReader(line.decode('utf-8') for line in response)
yield from reader
# @block-end parse-csv
# @block-start upload-vectors
from qdrant_client.http.models import PointStruct, Document
import uuid
csv_url = 'https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv'
# Retrieve a list of existing shard keys in the collection
existing_shard_keys = list(client.list_shard_keys(collection_name=collection_name).shard_keys)
dense_model = "sentence-transformers/all-MiniLM-L6-v2"
batch_size = 100
current_date = None
buffer: list[PointStruct] = []
for row in parse_csv(csv_url):
shard_date = row['datetime'][:10] # Extract YYYY-MM-DD
if shard_date != current_date:
# Flush buffer for the previous date before switching
if buffer:
client.upload_points(
collection_name=collection_name,
points=buffer,
shard_key_selector=current_date,
)
buffer = []
# Create shard for the new date if it doesn't exist yet
if shard_date not in existing_shard_keys:
client.create_shard_key(collection_name, shard_date)
existing_shard_keys.append(shard_date)
current_date = shard_date
# Add point to buffer
buffer.append(PointStruct(
id=uuid.uuid4().hex,
payload={"text": row['text'], "datetime": row['datetime']},
vector={"dense_vector": Document(text=row["text"], model=dense_model)}
))
# Flush batch if buffer size exceeds batch size
if len(buffer) >= batch_size:
client.upload_points(
collection_name=collection_name,
points=buffer,
shard_key_selector=current_date,
)
buffer = []
# Flush remaining partial batch
if buffer:
client.upload_points(
collection_name=collection_name,
points=buffer,
shard_key_selector=current_date,
)
# @block-end upload-vectors
# @block-start search-single-shard
query_text = "coffee"
resp = client.query_points(
collection_name=collection_name,
query=Document(text=query_text, model=dense_model),
using="dense_vector",
limit=5,
shard_key_selector="2026-04-07"
)
print(resp)
# @block-end search-single-shard
# @block-start search-multiple-shards
resp = client.query_points(
collection_name=collection_name,
query=Document(text=query_text, model=dense_model),
using="dense_vector",
limit=5,
shard_key_selector=["2026-04-06","2026-04-07"]
)
print(resp)
# @block-end search-multiple-shards
# @block-start search-all-shards
resp = client.query_points(
collection_name=collection_name,
query=Document(text=query_text, model=dense_model),
using="dense_vector",
limit=5,
)
print(resp)
# @block-end search-all-shards
# @block-start pruning-shards
from datetime import date, timedelta
today = "2026-04-08"
oldest_shard_key = (date.fromisoformat(today) - timedelta(days=7)).isoformat()
client.create_shard_key(collection_name, today)
client.delete_shard_key(collection_name, oldest_shard_key)
# @block-end pruning-shards
# @block-start ingest-new-data
client.upsert(
collection_name=collection_name,
points=[PointStruct(
id=uuid.uuid4().hex,
payload={"text": "The best way to start a Wednesday is with a cup of coffee", "datetime": "2026-04-08T07:57:47"},
vector={
"dense_vector": Document(text="The best way to start a Wednesday is with a cup of coffee", model=dense_model)
})],
shard_key_selector=today
)
# @block-end ingest-new-data
@@ -0,0 +1,260 @@
use std::collections::{HashMap, HashSet};
use chrono::NaiveDate;
use qdrant_client::Qdrant;
use qdrant_client::qdrant::{
CreateCollectionBuilder, CreateShardKeyBuilder, CreateShardKeyRequestBuilder,
DeleteShardKeyRequestBuilder, Distance, Document, DocumentBuilder,
PointStruct, Query, QueryPointsBuilder, ShardKeySelector, ShardingMethod,
UpsertPointsBuilder, VectorParamsBuilder, VectorsConfigBuilder, shard_key,
};
pub async fn main() -> anyhow::Result<()> {
// @hide-start
let QDRANT_URL = "";
let QDRANT_API_KEY = "";
// @hide-end
// @block-start initialize-client
let client = Qdrant::from_url(QDRANT_URL)
.api_key(QDRANT_API_KEY)
.build()?;
// @block-end initialize-client
// @block-start create-collection
let collection_name = "my_collection";
if client.collection_exists(collection_name).await? {
client.delete_collection(collection_name).await?;
}
let mut vectors_config = VectorsConfigBuilder::default();
vectors_config.add_named_vector_params(
"dense_vector",
VectorParamsBuilder::new(384, Distance::Cosine),
);
client
.create_collection(
CreateCollectionBuilder::new(collection_name)
.vectors_config(vectors_config)
.sharding_method(ShardingMethod::Custom.into()),
)
.await?;
// @block-end create-collection
// @block-start parse-csv
struct CsvRow {
text: String,
datetime: String,
}
fn parse_csv(url: &str) -> anyhow::Result<impl Iterator<Item = anyhow::Result<CsvRow>>> {
let reader = ureq::get(url).call()?.into_body().into_reader();
let mut rdr = csv::Reader::from_reader(reader);
let headers = rdr.headers()?.clone();
let text_idx = headers.iter().position(|h| h == "text").unwrap();
let datetime_idx = headers.iter().position(|h| h == "datetime").unwrap();
let iter = rdr.into_records().map(move |result| {
let record = result?;
Ok(CsvRow {
text: record[text_idx].to_string(),
datetime: record[datetime_idx].to_string(),
})
});
Ok(iter)
}
// @block-end parse-csv
// @block-start upload-vectors
let csv_url = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv";
// Retrieve a list of existing shard keys in the collection
let response = client.list_shard_keys(collection_name).await?;
let mut existing_shard_keys: HashSet<String> = response
.shard_keys
.into_iter()
.filter_map(|d| {
d.key?.key.and_then(|k| match k {
shard_key::Key::Keyword(s) => Some(s),
_ => None,
})
})
.collect();
let dense_model = "sentence-transformers/all-MiniLM-L6-v2";
let batch_size = 100;
let mut current_date = String::new();
let mut buffer: Vec<PointStruct> = Vec::new();
for row in parse_csv(csv_url)? {
let row = row?;
let text = row.text;
let datetime = row.datetime;
let shard_date = datetime[..10].to_string(); // Extract YYYY-MM-DD
if shard_date != current_date {
// Flush buffer for the previous date before switching
if !buffer.is_empty() {
client
.upsert_points(
UpsertPointsBuilder::new(
collection_name,
std::mem::take(&mut buffer),
)
.shard_key_selector(current_date.clone()),
)
.await?;
}
// Create shard for the new date if it doesn't exist yet
if !existing_shard_keys.contains(&shard_date) {
client
.create_shard_key(
CreateShardKeyRequestBuilder::new(collection_name).request(
CreateShardKeyBuilder::default().shard_key(shard_date.clone()),
),
)
.await?;
existing_shard_keys.insert(shard_date.clone());
}
current_date = shard_date;
}
// Add point to buffer
buffer.push(PointStruct::new(
uuid::Uuid::new_v4().to_string(),
HashMap::from([(
"dense_vector".to_string(),
DocumentBuilder::new(&text, dense_model).build(),
)]),
[("text", text.into()), ("datetime", datetime.into())],
));
// Flush batch if buffer size exceeds batch size
if buffer.len() >= batch_size {
client
.upsert_points(
UpsertPointsBuilder::new(collection_name, std::mem::take(&mut buffer))
.shard_key_selector(current_date.clone()),
)
.await?;
}
}
// Flush remaining partial batch
if !buffer.is_empty() {
client
.upsert_points(
UpsertPointsBuilder::new(collection_name, buffer)
.shard_key_selector(current_date.clone()),
)
.await?;
}
// @block-end upload-vectors
// @block-start search-single-shard
let query_text = "coffee";
let result = client
.query(
QueryPointsBuilder::new(collection_name)
.query(Query::new_nearest(Document::new(query_text, dense_model)))
.using("dense_vector")
.limit(5)
.shard_key_selector("2026-04-07".to_string()),
)
.await?;
for hit in result.result {
println!("{:?}", hit);
}
// @block-end search-single-shard
// @block-start search-multiple-shards
let result = client
.query(
QueryPointsBuilder::new(collection_name)
.query(Query::new_nearest(Document::new(query_text, dense_model)))
.using("dense_vector")
.limit(5)
.shard_key_selector(ShardKeySelector {
shard_keys: vec![
"2026-04-06".to_string().into(),
"2026-04-07".to_string().into(),
],
fallback: None,
}),
)
.await?;
for hit in result.result {
println!("{:?}", hit);
}
// @block-end search-multiple-shards
// @block-start search-all-shards
let result = client
.query(
QueryPointsBuilder::new(collection_name)
.query(Query::new_nearest(Document::new(query_text, dense_model)))
.using("dense_vector")
.limit(5),
)
.await?;
for hit in result.result {
println!("{:?}", hit);
}
// @block-end search-all-shards
// @block-start pruning-shards
let today = "2026-04-08";
let oldest_shard_key = (NaiveDate::parse_from_str(today, "%Y-%m-%d")?
- chrono::Duration::days(7))
.to_string();
client
.create_shard_key(
CreateShardKeyRequestBuilder::new(collection_name)
.request(CreateShardKeyBuilder::default().shard_key(today.to_string())),
)
.await?;
client
.delete_shard_key(
DeleteShardKeyRequestBuilder::new(collection_name)
.key(shard_key::Key::Keyword(oldest_shard_key)),
)
.await?;
// @block-end pruning-shards
// @block-start ingest-new-data
client
.upsert_points(
UpsertPointsBuilder::new(
collection_name,
vec![PointStruct::new(
uuid::Uuid::new_v4().to_string(),
HashMap::from([(
"dense_vector".to_string(),
DocumentBuilder::new(
"The best way to start a Wednesday is with a cup of coffee",
dense_model,
)
.build(),
)]),
[
("text", "The best way to start a Wednesday is with a cup of coffee".into()),
("datetime", "2026-04-08T07:57:47".into()),
],
)],
)
.shard_key_selector(today.to_string()),
)
.await?;
// @block-end ingest-new-data
Ok(())
}
@@ -0,0 +1,210 @@
import { QdrantClient } from "@qdrant/js-client-rest";
// @hide-start
const QDRANT_URL = "";
const QDRANT_API_KEY = "";
// @hide-end
// @block-start initialize-client
const client = new QdrantClient({
url: QDRANT_URL,
apiKey: QDRANT_API_KEY,
});
// @block-end initialize-client
// @block-start create-collection
const collectionName = "my_collection";
if (await client.collectionExists(collectionName)) {
await client.deleteCollection(collectionName);
}
await client.createCollection(collectionName, {
vectors: {
dense_vector: {
size: 384,
distance: "Cosine",
},
},
sharding_method: "custom",
});
// @block-end create-collection
// @block-start parse-csv
function parseCsvLine(line: string): string[] {
const fields: string[] = [];
let i = 0;
while (i < line.length) {
if (line[i] === '"') {
i++;
let field = "";
while (i < line.length) {
if (line[i] === '"' && line[i + 1] === '"') { field += '"'; i += 2; }
else if (line[i] === '"') { i++; break; }
else { field += line[i++]; }
}
fields.push(field);
if (line[i] === ",") i++;
} else {
const start = i;
while (i < line.length && line[i] !== ",") i++;
fields.push(line.slice(start, i));
if (i < line.length) i++;
}
}
return fields;
}
async function* parseCSV(url: string): AsyncGenerator<{ text: string; datetime: string }> {
const response = await fetch(url);
const reader = response.body!.getReader();
const decoder = new TextDecoder();
let remainder = "";
let headers: string[] | null = null;
let textIdx = -1;
let datetimeIdx = -1;
while (true) {
const { done, value } = await reader.read();
const chunk = done ? "" : decoder.decode(value, { stream: true });
const lines = (remainder + chunk).split("\n");
remainder = done ? "" : lines.pop()!;
for (const line of lines) {
if (!line.trim()) continue;
if (headers === null) {
headers = line.split(",");
textIdx = headers.indexOf("text");
datetimeIdx = headers.indexOf("datetime");
continue;
}
const fields = parseCsvLine(line);
yield { text: fields[textIdx], datetime: fields[datetimeIdx] };
}
if (done) break;
}
}
// @block-end parse-csv
// @block-start upload-vectors
const csvUrl = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv";
// Retrieve a list of existing shard keys in the collection
const shardKeysResult = await client.listShardKeys(collectionName);
const existingShardKeys = new Set((shardKeysResult.shard_keys ?? []).map((d) => String(d.key)));
const denseModel = "sentence-transformers/all-MiniLM-L6-v2";
const batchSize = 100;
let currentDate = "";
let buffer: Extract<Parameters<typeof client.upsert>[1], { points: unknown }>['points'] = [];
for await (const { text: postText, datetime } of parseCSV(csvUrl)) {
const shardDate = datetime.slice(0, 10); // Extract YYYY-MM-DD
if (shardDate !== currentDate) {
// Flush buffer for the previous date before switching
if (buffer.length > 0) {
await client.upsert(collectionName, { points: buffer, shard_key: currentDate });
buffer = [];
}
// Create shard for the new date if it doesn't exist yet
if (!existingShardKeys.has(shardDate)) {
await client.createShardKey(collectionName, { shard_key: shardDate });
existingShardKeys.add(shardDate);
}
currentDate = shardDate;
}
// Add point to buffer
buffer.push({
id: crypto.randomUUID(),
vector: { dense_vector: { text: postText, model: denseModel } },
payload: { text: postText, datetime },
});
// Flush batch if buffer size exceeds batch size
if (buffer.length >= batchSize) {
await client.upsert(collectionName, { points: buffer, shard_key: currentDate });
buffer = [];
}
}
// Flush remaining partial batch
if (buffer.length > 0) {
await client.upsert(collectionName, { points: buffer, shard_key: currentDate });
}
// @block-end upload-vectors
// @block-start search-single-shard
const queryText = "coffee";
const singleShardResult = await client.query(collectionName, {
query: { text: queryText, model: denseModel },
using: "dense_vector",
limit: 5,
shard_key: "2026-04-07",
});
for (const hit of singleShardResult.points) {
console.log(hit);
}
// @block-end search-single-shard
// @block-start search-multiple-shards
const multiShardResult = await client.query(collectionName, {
query: { text: queryText, model: denseModel },
using: "dense_vector",
limit: 5,
shard_key: ["2026-04-06", "2026-04-07"],
});
for (const hit of multiShardResult.points) {
console.log(hit);
}
// @block-end search-multiple-shards
// @block-start search-all-shards
const allShardsResult = await client.query(collectionName, {
query: { text: queryText, model: denseModel },
using: "dense_vector",
limit: 5,
});
for (const hit of allShardsResult.points) {
console.log(hit);
}
// @block-end search-all-shards
// @block-start pruning-shards
const today = "2026-04-08";
const oldestDate = new Date(today);
oldestDate.setDate(oldestDate.getDate() - 7);
const oldestShardKey = oldestDate.toISOString().slice(0, 10);
await client.createShardKey(collectionName, { shard_key: today });
await client.deleteShardKey(collectionName, { shard_key: oldestShardKey });
// @block-end pruning-shards
// @block-start ingest-new-data
await client.upsert(collectionName, {
points: [
{
id: crypto.randomUUID(),
vector: {
dense_vector: {
text: "The best way to start a Wednesday is with a cup of coffee",
model: denseModel,
},
},
payload: {
text: "The best way to start a Wednesday is with a cup of coffee",
datetime: "2026-04-08T07:57:47",
},
},
],
shard_key: today,
});
// @block-end ingest-new-data