mirror of
https://github.com/qdrant/landing_page.git
synced 2026-10-09 21:08:31 +02:00
Time based sharding tutorial (#1805)
* add snippets * add snippets * add snippets * add snippets descriptions * Initial commit * Updated tutorial * Expand on shard_number * Add sentence about optimal batch size * Show how to query multiple shards * Small edits * Fixes * Add to landing page; adjust weight * Clean up code snippets * Don't set shard_key in code snippet * Review feedback * Update links to moved pages * Add C# code snippets * Add Go code snippets * Add Java code snippets * Add Rust code snippets * Add TS code snippets * Add note about changing collection settings * Move CSV parsing to separate function * Trigger Build * Remove unused dependency --------- Co-authored-by: Abdon Pijpelink <abdon.pijpelink@qdrant.com>
This commit is contained in:
co-authored by
Abdon Pijpelink
parent
6d041d78ba
commit
dc564bbab0
@@ -0,0 +1,238 @@
|
||||
using System.Net.Http;
|
||||
using Microsoft.VisualBasic.FileIO;
|
||||
using Qdrant.Client;
|
||||
using Qdrant.Client.Grpc;
|
||||
|
||||
public class Snippet
|
||||
{
|
||||
public static async Task Run()
|
||||
{
|
||||
// @hide-start
|
||||
string QDRANT_URL = "";
|
||||
string QDRANT_API_KEY = "";
|
||||
// @hide-end
|
||||
|
||||
// @block-start initialize-client
|
||||
var client = new QdrantClient(
|
||||
host: QDRANT_URL,
|
||||
https: true,
|
||||
apiKey: QDRANT_API_KEY
|
||||
);
|
||||
// @block-end initialize-client
|
||||
|
||||
// @block-start create-collection
|
||||
string collectionName = "my_collection";
|
||||
|
||||
if (await client.CollectionExistsAsync(collectionName))
|
||||
await client.DeleteCollectionAsync(collectionName);
|
||||
|
||||
await client.CreateCollectionAsync(
|
||||
collectionName: collectionName,
|
||||
vectorsConfig: new VectorParamsMap
|
||||
{
|
||||
Map = {
|
||||
["dense_vector"] = new VectorParams { Size = 384, Distance = Distance.Cosine }
|
||||
}
|
||||
},
|
||||
shardingMethod: ShardingMethod.Custom
|
||||
);
|
||||
// @block-end create-collection
|
||||
|
||||
// @block-start parse-csv
|
||||
async IAsyncEnumerable<(string text, string datetime)> ParseCsv(string url)
|
||||
{
|
||||
using var httpClient = new HttpClient();
|
||||
using var stream = await httpClient.GetStreamAsync(url);
|
||||
using var parser = new TextFieldParser(new StreamReader(stream));
|
||||
parser.TextFieldType = Microsoft.VisualBasic.FileIO.FieldType.Delimited;
|
||||
parser.SetDelimiters(",");
|
||||
string[]? headers = parser.ReadFields();
|
||||
int textIdx = Array.IndexOf(headers!, "text");
|
||||
int datetimeIdx = Array.IndexOf(headers!, "datetime");
|
||||
while (!parser.EndOfData)
|
||||
{
|
||||
var fields = parser.ReadFields()!;
|
||||
yield return (fields[textIdx], fields[datetimeIdx]);
|
||||
}
|
||||
}
|
||||
// @block-end parse-csv
|
||||
|
||||
// @block-start upload-vectors
|
||||
string csvUrl = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv";
|
||||
|
||||
// Retrieve a list of existing shard keys in the collection
|
||||
var existingShardKeys = (await client.ListShardKeysAsync(collectionName))
|
||||
.Select(sk => sk.Key.Keyword)
|
||||
.ToHashSet();
|
||||
|
||||
string denseModel = "sentence-transformers/all-MiniLM-L6-v2";
|
||||
int batchSize = 100;
|
||||
string? currentDate = null;
|
||||
var buffer = new List<PointStruct>();
|
||||
|
||||
await foreach (var (text, datetime) in ParseCsv(csvUrl))
|
||||
{
|
||||
string shardDate = datetime[..10]; // Extract YYYY-MM-DD
|
||||
|
||||
if (shardDate != currentDate)
|
||||
{
|
||||
// Flush buffer for the previous date before switching
|
||||
if (buffer.Count > 0)
|
||||
{
|
||||
await client.UpsertAsync(
|
||||
collectionName: collectionName,
|
||||
points: buffer,
|
||||
shardKeySelector: new ShardKeySelector
|
||||
{
|
||||
ShardKeys = { new List<ShardKey> { currentDate! } }
|
||||
}
|
||||
);
|
||||
buffer.Clear();
|
||||
}
|
||||
|
||||
// Create shard for the new date if it doesn't exist yet
|
||||
if (!existingShardKeys.Contains(shardDate))
|
||||
{
|
||||
await client.CreateShardKeyAsync(
|
||||
collectionName,
|
||||
new CreateShardKey { ShardKey = new ShardKey { Keyword = shardDate } }
|
||||
);
|
||||
existingShardKeys.Add(shardDate);
|
||||
}
|
||||
|
||||
currentDate = shardDate;
|
||||
}
|
||||
|
||||
// Add point to buffer
|
||||
buffer.Add(new PointStruct
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Vectors = new Dictionary<string, Vector>
|
||||
{
|
||||
["dense_vector"] = new Document { Text = text, Model = denseModel }
|
||||
},
|
||||
Payload = { ["text"] = text, ["datetime"] = datetime }
|
||||
});
|
||||
|
||||
// Flush batch if buffer size exceeds batch size
|
||||
if (buffer.Count >= batchSize)
|
||||
{
|
||||
await client.UpsertAsync(
|
||||
collectionName: collectionName,
|
||||
points: buffer,
|
||||
shardKeySelector: new ShardKeySelector
|
||||
{
|
||||
ShardKeys = { new List<ShardKey> { currentDate! } }
|
||||
}
|
||||
);
|
||||
buffer.Clear();
|
||||
}
|
||||
}
|
||||
|
||||
// Flush remaining partial batch
|
||||
if (buffer.Count > 0)
|
||||
{
|
||||
await client.UpsertAsync(
|
||||
collectionName: collectionName,
|
||||
points: buffer,
|
||||
shardKeySelector: new ShardKeySelector
|
||||
{
|
||||
ShardKeys = { new List<ShardKey> { currentDate! } }
|
||||
}
|
||||
);
|
||||
}
|
||||
// @block-end upload-vectors
|
||||
|
||||
// @block-start search-single-shard
|
||||
string queryText = "coffee";
|
||||
|
||||
var result = await client.QueryAsync(
|
||||
collectionName: collectionName,
|
||||
query: new Document { Text = queryText, Model = denseModel },
|
||||
usingVector: "dense_vector",
|
||||
limit: 5,
|
||||
shardKeySelector: new ShardKeySelector
|
||||
{
|
||||
ShardKeys = { new List<ShardKey> { "2026-04-07" } }
|
||||
}
|
||||
);
|
||||
|
||||
foreach (var hit in result)
|
||||
Console.WriteLine(hit);
|
||||
// @block-end search-single-shard
|
||||
|
||||
// @block-start search-multiple-shards
|
||||
result = await client.QueryAsync(
|
||||
collectionName: collectionName,
|
||||
query: new Document { Text = queryText, Model = denseModel },
|
||||
usingVector: "dense_vector",
|
||||
limit: 5,
|
||||
shardKeySelector: new ShardKeySelector
|
||||
{
|
||||
ShardKeys = { new List<ShardKey> { "2026-04-06", "2026-04-07" } }
|
||||
}
|
||||
);
|
||||
|
||||
foreach (var hit in result)
|
||||
Console.WriteLine(hit);
|
||||
// @block-end search-multiple-shards
|
||||
|
||||
// @block-start search-all-shards
|
||||
result = await client.QueryAsync(
|
||||
collectionName: collectionName,
|
||||
query: new Document { Text = queryText, Model = denseModel },
|
||||
usingVector: "dense_vector",
|
||||
limit: 5
|
||||
);
|
||||
|
||||
foreach (var hit in result)
|
||||
Console.WriteLine(hit);
|
||||
// @block-end search-all-shards
|
||||
|
||||
// @block-start pruning-shards
|
||||
string today = "2026-04-08";
|
||||
string oldestShardKey = DateOnly.ParseExact(today, "yyyy-MM-dd")
|
||||
.AddDays(-7)
|
||||
.ToString("yyyy-MM-dd");
|
||||
|
||||
await client.CreateShardKeyAsync(
|
||||
collectionName,
|
||||
new CreateShardKey { ShardKey = new ShardKey { Keyword = today } }
|
||||
);
|
||||
await client.DeleteShardKeyAsync(
|
||||
collectionName,
|
||||
new DeleteShardKey { ShardKey = new ShardKey { Keyword = oldestShardKey } }
|
||||
);
|
||||
// @block-end pruning-shards
|
||||
|
||||
// @block-start ingest-new-data
|
||||
await client.UpsertAsync(
|
||||
collectionName: collectionName,
|
||||
points: new List<PointStruct>
|
||||
{
|
||||
new()
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Vectors = new Dictionary<string, Vector>
|
||||
{
|
||||
["dense_vector"] = new Document
|
||||
{
|
||||
Text = "The best way to start a Wednesday is with a cup of coffee",
|
||||
Model = denseModel
|
||||
}
|
||||
},
|
||||
Payload =
|
||||
{
|
||||
["text"] = "The best way to start a Wednesday is with a cup of coffee",
|
||||
["datetime"] = "2026-04-08T07:57:47"
|
||||
}
|
||||
}
|
||||
},
|
||||
shardKeySelector: new ShardKeySelector
|
||||
{
|
||||
ShardKeys = { new List<ShardKey> { today } }
|
||||
}
|
||||
);
|
||||
// @block-end ingest-new-data
|
||||
}
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
```csharp
|
||||
string collectionName = "my_collection";
|
||||
|
||||
if (await client.CollectionExistsAsync(collectionName))
|
||||
await client.DeleteCollectionAsync(collectionName);
|
||||
|
||||
await client.CreateCollectionAsync(
|
||||
collectionName: collectionName,
|
||||
vectorsConfig: new VectorParamsMap
|
||||
{
|
||||
Map = {
|
||||
["dense_vector"] = new VectorParams { Size = 384, Distance = Distance.Cosine }
|
||||
}
|
||||
},
|
||||
shardingMethod: ShardingMethod.Custom
|
||||
);
|
||||
```
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
```go
|
||||
collectionName := "my_collection"
|
||||
|
||||
exists, err := client.CollectionExists(context.Background(), collectionName)
|
||||
if exists {
|
||||
client.DeleteCollection(context.Background(), collectionName)
|
||||
}
|
||||
|
||||
client.CreateCollection(context.Background(), &qdrant.CreateCollection{
|
||||
CollectionName: collectionName,
|
||||
VectorsConfig: qdrant.NewVectorsConfigMap(
|
||||
map[string]*qdrant.VectorParams{
|
||||
"dense_vector": {
|
||||
Size: 384,
|
||||
Distance: qdrant.Distance_Cosine,
|
||||
},
|
||||
},
|
||||
),
|
||||
ShardingMethod: qdrant.ShardingMethod_Custom.Enum(),
|
||||
})
|
||||
```
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
```java
|
||||
String collectionName = "my_collection";
|
||||
|
||||
if (client.collectionExistsAsync(collectionName).get()) {
|
||||
client.deleteCollectionAsync(collectionName).get();
|
||||
}
|
||||
|
||||
client.createCollectionAsync(
|
||||
CreateCollection.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.setVectorsConfig(VectorsConfig.newBuilder().setParamsMap(
|
||||
VectorParamsMap.newBuilder().putAllMap(Map.of(
|
||||
"dense_vector",
|
||||
VectorParams.newBuilder()
|
||||
.setSize(384)
|
||||
.setDistance(Distance.Cosine)
|
||||
.build()))))
|
||||
.setShardingMethod(ShardingMethod.Custom)
|
||||
.build()
|
||||
).get();
|
||||
```
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
```python
|
||||
from qdrant_client import models
|
||||
|
||||
collection_name = "my_collection"
|
||||
|
||||
if client.collection_exists(collection_name=collection_name):
|
||||
client.delete_collection(collection_name=collection_name)
|
||||
|
||||
client.create_collection(
|
||||
collection_name=collection_name,
|
||||
vectors_config={
|
||||
"dense_vector": models.VectorParams(
|
||||
size=384, distance=models.Distance.COSINE
|
||||
)
|
||||
},
|
||||
sharding_method=models.ShardingMethod.CUSTOM
|
||||
)
|
||||
```
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
```rust
|
||||
let collection_name = "my_collection";
|
||||
|
||||
if client.collection_exists(collection_name).await? {
|
||||
client.delete_collection(collection_name).await?;
|
||||
}
|
||||
|
||||
let mut vectors_config = VectorsConfigBuilder::default();
|
||||
vectors_config.add_named_vector_params(
|
||||
"dense_vector",
|
||||
VectorParamsBuilder::new(384, Distance::Cosine),
|
||||
);
|
||||
|
||||
client
|
||||
.create_collection(
|
||||
CreateCollectionBuilder::new(collection_name)
|
||||
.vectors_config(vectors_config)
|
||||
.sharding_method(ShardingMethod::Custom.into()),
|
||||
)
|
||||
.await?;
|
||||
```
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
```typescript
|
||||
const collectionName = "my_collection";
|
||||
|
||||
if (await client.collectionExists(collectionName)) {
|
||||
await client.deleteCollection(collectionName);
|
||||
}
|
||||
|
||||
await client.createCollection(collectionName, {
|
||||
vectors: {
|
||||
dense_vector: {
|
||||
size: 384,
|
||||
distance: "Cosine",
|
||||
},
|
||||
},
|
||||
sharding_method: "custom",
|
||||
});
|
||||
```
|
||||
+211
@@ -0,0 +1,211 @@
|
||||
```csharp
|
||||
using System.Net.Http;
|
||||
using Microsoft.VisualBasic.FileIO;
|
||||
using Qdrant.Client;
|
||||
using Qdrant.Client.Grpc;
|
||||
|
||||
var client = new QdrantClient(
|
||||
host: QDRANT_URL,
|
||||
https: true,
|
||||
apiKey: QDRANT_API_KEY
|
||||
);
|
||||
|
||||
string collectionName = "my_collection";
|
||||
|
||||
if (await client.CollectionExistsAsync(collectionName))
|
||||
await client.DeleteCollectionAsync(collectionName);
|
||||
|
||||
await client.CreateCollectionAsync(
|
||||
collectionName: collectionName,
|
||||
vectorsConfig: new VectorParamsMap
|
||||
{
|
||||
Map = {
|
||||
["dense_vector"] = new VectorParams { Size = 384, Distance = Distance.Cosine }
|
||||
}
|
||||
},
|
||||
shardingMethod: ShardingMethod.Custom
|
||||
);
|
||||
|
||||
async IAsyncEnumerable<(string text, string datetime)> ParseCsv(string url)
|
||||
{
|
||||
using var httpClient = new HttpClient();
|
||||
using var stream = await httpClient.GetStreamAsync(url);
|
||||
using var parser = new TextFieldParser(new StreamReader(stream));
|
||||
parser.TextFieldType = Microsoft.VisualBasic.FileIO.FieldType.Delimited;
|
||||
parser.SetDelimiters(",");
|
||||
string[]? headers = parser.ReadFields();
|
||||
int textIdx = Array.IndexOf(headers!, "text");
|
||||
int datetimeIdx = Array.IndexOf(headers!, "datetime");
|
||||
while (!parser.EndOfData)
|
||||
{
|
||||
var fields = parser.ReadFields()!;
|
||||
yield return (fields[textIdx], fields[datetimeIdx]);
|
||||
}
|
||||
}
|
||||
|
||||
string csvUrl = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv";
|
||||
|
||||
// Retrieve a list of existing shard keys in the collection
|
||||
var existingShardKeys = (await client.ListShardKeysAsync(collectionName))
|
||||
.Select(sk => sk.Key.Keyword)
|
||||
.ToHashSet();
|
||||
|
||||
string denseModel = "sentence-transformers/all-MiniLM-L6-v2";
|
||||
int batchSize = 100;
|
||||
string? currentDate = null;
|
||||
var buffer = new List<PointStruct>();
|
||||
|
||||
await foreach (var (text, datetime) in ParseCsv(csvUrl))
|
||||
{
|
||||
string shardDate = datetime[..10]; // Extract YYYY-MM-DD
|
||||
|
||||
if (shardDate != currentDate)
|
||||
{
|
||||
// Flush buffer for the previous date before switching
|
||||
if (buffer.Count > 0)
|
||||
{
|
||||
await client.UpsertAsync(
|
||||
collectionName: collectionName,
|
||||
points: buffer,
|
||||
shardKeySelector: new ShardKeySelector
|
||||
{
|
||||
ShardKeys = { new List<ShardKey> { currentDate! } }
|
||||
}
|
||||
);
|
||||
buffer.Clear();
|
||||
}
|
||||
|
||||
// Create shard for the new date if it doesn't exist yet
|
||||
if (!existingShardKeys.Contains(shardDate))
|
||||
{
|
||||
await client.CreateShardKeyAsync(
|
||||
collectionName,
|
||||
new CreateShardKey { ShardKey = new ShardKey { Keyword = shardDate } }
|
||||
);
|
||||
existingShardKeys.Add(shardDate);
|
||||
}
|
||||
|
||||
currentDate = shardDate;
|
||||
}
|
||||
|
||||
// Add point to buffer
|
||||
buffer.Add(new PointStruct
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Vectors = new Dictionary<string, Vector>
|
||||
{
|
||||
["dense_vector"] = new Document { Text = text, Model = denseModel }
|
||||
},
|
||||
Payload = { ["text"] = text, ["datetime"] = datetime }
|
||||
});
|
||||
|
||||
// Flush batch if buffer size exceeds batch size
|
||||
if (buffer.Count >= batchSize)
|
||||
{
|
||||
await client.UpsertAsync(
|
||||
collectionName: collectionName,
|
||||
points: buffer,
|
||||
shardKeySelector: new ShardKeySelector
|
||||
{
|
||||
ShardKeys = { new List<ShardKey> { currentDate! } }
|
||||
}
|
||||
);
|
||||
buffer.Clear();
|
||||
}
|
||||
}
|
||||
|
||||
// Flush remaining partial batch
|
||||
if (buffer.Count > 0)
|
||||
{
|
||||
await client.UpsertAsync(
|
||||
collectionName: collectionName,
|
||||
points: buffer,
|
||||
shardKeySelector: new ShardKeySelector
|
||||
{
|
||||
ShardKeys = { new List<ShardKey> { currentDate! } }
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
string queryText = "coffee";
|
||||
|
||||
var result = await client.QueryAsync(
|
||||
collectionName: collectionName,
|
||||
query: new Document { Text = queryText, Model = denseModel },
|
||||
usingVector: "dense_vector",
|
||||
limit: 5,
|
||||
shardKeySelector: new ShardKeySelector
|
||||
{
|
||||
ShardKeys = { new List<ShardKey> { "2026-04-07" } }
|
||||
}
|
||||
);
|
||||
|
||||
foreach (var hit in result)
|
||||
Console.WriteLine(hit);
|
||||
|
||||
result = await client.QueryAsync(
|
||||
collectionName: collectionName,
|
||||
query: new Document { Text = queryText, Model = denseModel },
|
||||
usingVector: "dense_vector",
|
||||
limit: 5,
|
||||
shardKeySelector: new ShardKeySelector
|
||||
{
|
||||
ShardKeys = { new List<ShardKey> { "2026-04-06", "2026-04-07" } }
|
||||
}
|
||||
);
|
||||
|
||||
foreach (var hit in result)
|
||||
Console.WriteLine(hit);
|
||||
|
||||
result = await client.QueryAsync(
|
||||
collectionName: collectionName,
|
||||
query: new Document { Text = queryText, Model = denseModel },
|
||||
usingVector: "dense_vector",
|
||||
limit: 5
|
||||
);
|
||||
|
||||
foreach (var hit in result)
|
||||
Console.WriteLine(hit);
|
||||
|
||||
string today = "2026-04-08";
|
||||
string oldestShardKey = DateOnly.ParseExact(today, "yyyy-MM-dd")
|
||||
.AddDays(-7)
|
||||
.ToString("yyyy-MM-dd");
|
||||
|
||||
await client.CreateShardKeyAsync(
|
||||
collectionName,
|
||||
new CreateShardKey { ShardKey = new ShardKey { Keyword = today } }
|
||||
);
|
||||
await client.DeleteShardKeyAsync(
|
||||
collectionName,
|
||||
new DeleteShardKey { ShardKey = new ShardKey { Keyword = oldestShardKey } }
|
||||
);
|
||||
|
||||
await client.UpsertAsync(
|
||||
collectionName: collectionName,
|
||||
points: new List<PointStruct>
|
||||
{
|
||||
new()
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Vectors = new Dictionary<string, Vector>
|
||||
{
|
||||
["dense_vector"] = new Document
|
||||
{
|
||||
Text = "The best way to start a Wednesday is with a cup of coffee",
|
||||
Model = denseModel
|
||||
}
|
||||
},
|
||||
Payload =
|
||||
{
|
||||
["text"] = "The best way to start a Wednesday is with a cup of coffee",
|
||||
["datetime"] = "2026-04-08T07:57:47"
|
||||
}
|
||||
}
|
||||
},
|
||||
shardKeySelector: new ShardKeySelector
|
||||
{
|
||||
ShardKeys = { new List<ShardKey> { today } }
|
||||
}
|
||||
);
|
||||
```
|
||||
+240
@@ -0,0 +1,240 @@
|
||||
```go
|
||||
import (
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/qdrant/go-client/qdrant"
|
||||
)
|
||||
|
||||
type CSVRow struct {
|
||||
Text string
|
||||
Datetime string
|
||||
}
|
||||
|
||||
func parseCSV(url string, fn func(CSVRow)) error {
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
csvReader := csv.NewReader(resp.Body)
|
||||
headers, err := csvReader.Read()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
textIdx, datetimeIdx := -1, -1
|
||||
for i, h := range headers {
|
||||
switch h {
|
||||
case "text":
|
||||
textIdx = i
|
||||
case "datetime":
|
||||
datetimeIdx = i
|
||||
}
|
||||
}
|
||||
|
||||
for {
|
||||
row, err := csvReader.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fn(CSVRow{Text: row[textIdx], Datetime: row[datetimeIdx]})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
client, err := qdrant.NewClient(&qdrant.Config{
|
||||
Host: QDRANT_URL,
|
||||
APIKey: QDRANT_API_KEY,
|
||||
UseTLS: true,
|
||||
})
|
||||
|
||||
collectionName := "my_collection"
|
||||
|
||||
exists, err := client.CollectionExists(context.Background(), collectionName)
|
||||
if exists {
|
||||
client.DeleteCollection(context.Background(), collectionName)
|
||||
}
|
||||
|
||||
client.CreateCollection(context.Background(), &qdrant.CreateCollection{
|
||||
CollectionName: collectionName,
|
||||
VectorsConfig: qdrant.NewVectorsConfigMap(
|
||||
map[string]*qdrant.VectorParams{
|
||||
"dense_vector": {
|
||||
Size: 384,
|
||||
Distance: qdrant.Distance_Cosine,
|
||||
},
|
||||
},
|
||||
),
|
||||
ShardingMethod: qdrant.ShardingMethod_Custom.Enum(),
|
||||
})
|
||||
|
||||
csvUrl := "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv"
|
||||
|
||||
shardKeyDescriptions, err := client.ListShardKeys(context.Background(), collectionName)
|
||||
|
||||
// Retrieve a list of existing shard keys in the collection
|
||||
existingShardKeys := make(map[string]bool)
|
||||
for _, desc := range shardKeyDescriptions {
|
||||
existingShardKeys[desc.Key.GetKeyword()] = true
|
||||
}
|
||||
|
||||
denseModel := "sentence-transformers/all-MiniLM-L6-v2"
|
||||
batchSize := 100
|
||||
var currentDate string
|
||||
var buffer []*qdrant.PointStruct
|
||||
|
||||
err = parseCSV(csvUrl, func(row CSVRow) {
|
||||
text := row.Text
|
||||
datetime := row.Datetime
|
||||
shardDate := datetime[:10] // Extract YYYY-MM-DD
|
||||
|
||||
if shardDate != currentDate {
|
||||
// Flush buffer for the previous date before switching
|
||||
if len(buffer) > 0 {
|
||||
client.Upsert(context.Background(), &qdrant.UpsertPoints{
|
||||
CollectionName: collectionName,
|
||||
Points: buffer,
|
||||
ShardKeySelector: &qdrant.ShardKeySelector{
|
||||
ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey(currentDate)},
|
||||
},
|
||||
})
|
||||
buffer = nil
|
||||
}
|
||||
|
||||
// Create shard for the new date if it doesn't exist yet
|
||||
if !existingShardKeys[shardDate] {
|
||||
client.CreateShardKey(context.Background(), collectionName, &qdrant.CreateShardKey{
|
||||
ShardKey: qdrant.NewShardKey(shardDate),
|
||||
})
|
||||
existingShardKeys[shardDate] = true
|
||||
}
|
||||
|
||||
currentDate = shardDate
|
||||
}
|
||||
|
||||
// Add point to buffer
|
||||
buffer = append(buffer, &qdrant.PointStruct{
|
||||
Id: qdrant.NewID(uuid.New().String()),
|
||||
Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{
|
||||
"dense_vector": qdrant.NewVectorDocument(&qdrant.Document{
|
||||
Text: text,
|
||||
Model: denseModel,
|
||||
}),
|
||||
}),
|
||||
Payload: qdrant.NewValueMap(map[string]any{
|
||||
"text": text,
|
||||
"datetime": datetime,
|
||||
}),
|
||||
})
|
||||
|
||||
// Flush batch if buffer size exceeds batch size
|
||||
if len(buffer) >= batchSize {
|
||||
client.Upsert(context.Background(), &qdrant.UpsertPoints{
|
||||
CollectionName: collectionName,
|
||||
Points: buffer,
|
||||
ShardKeySelector: &qdrant.ShardKeySelector{
|
||||
ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey(currentDate)},
|
||||
},
|
||||
})
|
||||
buffer = nil
|
||||
}
|
||||
})
|
||||
|
||||
// Flush remaining partial batch
|
||||
if len(buffer) > 0 {
|
||||
client.Upsert(context.Background(), &qdrant.UpsertPoints{
|
||||
CollectionName: collectionName,
|
||||
Points: buffer,
|
||||
ShardKeySelector: &qdrant.ShardKeySelector{
|
||||
ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey(currentDate)},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
queryText := "coffee"
|
||||
|
||||
result, err := client.Query(context.Background(), &qdrant.QueryPoints{
|
||||
CollectionName: collectionName,
|
||||
Query: qdrant.NewQueryDocument(&qdrant.Document{Text: queryText, Model: denseModel}),
|
||||
Using: qdrant.PtrOf("dense_vector"),
|
||||
Limit: qdrant.PtrOf(uint64(5)),
|
||||
ShardKeySelector: &qdrant.ShardKeySelector{
|
||||
ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey("2026-04-07")},
|
||||
},
|
||||
})
|
||||
|
||||
for _, hit := range result {
|
||||
fmt.Println(hit)
|
||||
}
|
||||
|
||||
result, err = client.Query(context.Background(), &qdrant.QueryPoints{
|
||||
CollectionName: collectionName,
|
||||
Query: qdrant.NewQueryDocument(&qdrant.Document{Text: queryText, Model: denseModel}),
|
||||
Using: qdrant.PtrOf("dense_vector"),
|
||||
Limit: qdrant.PtrOf(uint64(5)),
|
||||
ShardKeySelector: &qdrant.ShardKeySelector{
|
||||
ShardKeys: []*qdrant.ShardKey{
|
||||
qdrant.NewShardKey("2026-04-06"),
|
||||
qdrant.NewShardKey("2026-04-07"),
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
for _, hit := range result {
|
||||
fmt.Println(hit)
|
||||
}
|
||||
|
||||
result, err = client.Query(context.Background(), &qdrant.QueryPoints{
|
||||
CollectionName: collectionName,
|
||||
Query: qdrant.NewQueryDocument(&qdrant.Document{Text: queryText, Model: denseModel}),
|
||||
Using: qdrant.PtrOf("dense_vector"),
|
||||
Limit: qdrant.PtrOf(uint64(5)),
|
||||
})
|
||||
|
||||
for _, hit := range result {
|
||||
fmt.Println(hit)
|
||||
}
|
||||
|
||||
today := "2026-04-08"
|
||||
t, _ := time.Parse("2006-01-02", today)
|
||||
oldestShardKey := t.AddDate(0, 0, -7).Format("2006-01-02")
|
||||
|
||||
client.CreateShardKey(context.Background(), collectionName, &qdrant.CreateShardKey{
|
||||
ShardKey: qdrant.NewShardKey(today),
|
||||
})
|
||||
client.DeleteShardKey(context.Background(), collectionName, &qdrant.DeleteShardKey{
|
||||
ShardKey: qdrant.NewShardKey(oldestShardKey),
|
||||
})
|
||||
|
||||
client.Upsert(context.Background(), &qdrant.UpsertPoints{
|
||||
CollectionName: collectionName,
|
||||
Points: []*qdrant.PointStruct{
|
||||
{
|
||||
Id: qdrant.NewID(uuid.New().String()),
|
||||
Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{
|
||||
"dense_vector": qdrant.NewVectorDocument(&qdrant.Document{
|
||||
Text: "The best way to start a Wednesday is with a cup of coffee",
|
||||
Model: denseModel,
|
||||
}),
|
||||
}),
|
||||
Payload: qdrant.NewValueMap(map[string]any{
|
||||
"text": "The best way to start a Wednesday is with a cup of coffee",
|
||||
"datetime": "2026-04-08T07:57:47",
|
||||
}),
|
||||
},
|
||||
},
|
||||
ShardKeySelector: &qdrant.ShardKeySelector{
|
||||
ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey(today)},
|
||||
},
|
||||
})
|
||||
```
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
```csharp
|
||||
await client.UpsertAsync(
|
||||
collectionName: collectionName,
|
||||
points: new List<PointStruct>
|
||||
{
|
||||
new()
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Vectors = new Dictionary<string, Vector>
|
||||
{
|
||||
["dense_vector"] = new Document
|
||||
{
|
||||
Text = "The best way to start a Wednesday is with a cup of coffee",
|
||||
Model = denseModel
|
||||
}
|
||||
},
|
||||
Payload =
|
||||
{
|
||||
["text"] = "The best way to start a Wednesday is with a cup of coffee",
|
||||
["datetime"] = "2026-04-08T07:57:47"
|
||||
}
|
||||
}
|
||||
},
|
||||
shardKeySelector: new ShardKeySelector
|
||||
{
|
||||
ShardKeys = { new List<ShardKey> { today } }
|
||||
}
|
||||
);
|
||||
```
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
```go
|
||||
client.Upsert(context.Background(), &qdrant.UpsertPoints{
|
||||
CollectionName: collectionName,
|
||||
Points: []*qdrant.PointStruct{
|
||||
{
|
||||
Id: qdrant.NewID(uuid.New().String()),
|
||||
Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{
|
||||
"dense_vector": qdrant.NewVectorDocument(&qdrant.Document{
|
||||
Text: "The best way to start a Wednesday is with a cup of coffee",
|
||||
Model: denseModel,
|
||||
}),
|
||||
}),
|
||||
Payload: qdrant.NewValueMap(map[string]any{
|
||||
"text": "The best way to start a Wednesday is with a cup of coffee",
|
||||
"datetime": "2026-04-08T07:57:47",
|
||||
}),
|
||||
},
|
||||
},
|
||||
ShardKeySelector: &qdrant.ShardKeySelector{
|
||||
ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey(today)},
|
||||
},
|
||||
})
|
||||
```
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
```java
|
||||
client.upsertAsync(
|
||||
UpsertPoints.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.addAllPoints(List.of(
|
||||
PointStruct.newBuilder()
|
||||
.setId(id(UUID.randomUUID()))
|
||||
.setVectors(namedVectors(Map.of(
|
||||
"dense_vector",
|
||||
vector(Document.newBuilder()
|
||||
.setText("The best way to start a Wednesday is with a cup of coffee")
|
||||
.setModel(denseModel)
|
||||
.build()))))
|
||||
.putAllPayload(Map.of(
|
||||
"text", value("The best way to start a Wednesday is with a cup of coffee"),
|
||||
"datetime", value("2026-04-08T07:57:47")))
|
||||
.build()))
|
||||
.setShardKeySelector(shardKeySelector(today))
|
||||
.build()
|
||||
).get();
|
||||
```
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
```python
|
||||
client.upsert(
|
||||
collection_name=collection_name,
|
||||
points=[PointStruct(
|
||||
id=uuid.uuid4().hex,
|
||||
payload={"text": "The best way to start a Wednesday is with a cup of coffee", "datetime": "2026-04-08T07:57:47"},
|
||||
vector={
|
||||
"dense_vector": Document(text="The best way to start a Wednesday is with a cup of coffee", model=dense_model)
|
||||
})],
|
||||
shard_key_selector=today
|
||||
)
|
||||
```
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
```rust
|
||||
client
|
||||
.upsert_points(
|
||||
UpsertPointsBuilder::new(
|
||||
collection_name,
|
||||
vec![PointStruct::new(
|
||||
uuid::Uuid::new_v4().to_string(),
|
||||
HashMap::from([(
|
||||
"dense_vector".to_string(),
|
||||
DocumentBuilder::new(
|
||||
"The best way to start a Wednesday is with a cup of coffee",
|
||||
dense_model,
|
||||
)
|
||||
.build(),
|
||||
)]),
|
||||
[
|
||||
("text", "The best way to start a Wednesday is with a cup of coffee".into()),
|
||||
("datetime", "2026-04-08T07:57:47".into()),
|
||||
],
|
||||
)],
|
||||
)
|
||||
.shard_key_selector(today.to_string()),
|
||||
)
|
||||
.await?;
|
||||
```
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
```typescript
|
||||
await client.upsert(collectionName, {
|
||||
points: [
|
||||
{
|
||||
id: crypto.randomUUID(),
|
||||
vector: {
|
||||
dense_vector: {
|
||||
text: "The best way to start a Wednesday is with a cup of coffee",
|
||||
model: denseModel,
|
||||
},
|
||||
},
|
||||
payload: {
|
||||
text: "The best way to start a Wednesday is with a cup of coffee",
|
||||
datetime: "2026-04-08T07:57:47",
|
||||
},
|
||||
},
|
||||
],
|
||||
shard_key: today,
|
||||
});
|
||||
```
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
```csharp
|
||||
var client = new QdrantClient(
|
||||
host: QDRANT_URL,
|
||||
https: true,
|
||||
apiKey: QDRANT_API_KEY
|
||||
);
|
||||
```
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
```go
|
||||
client, err := qdrant.NewClient(&qdrant.Config{
|
||||
Host: QDRANT_URL,
|
||||
APIKey: QDRANT_API_KEY,
|
||||
UseTLS: true,
|
||||
})
|
||||
```
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
```java
|
||||
QdrantClient client =
|
||||
new QdrantClient(
|
||||
QdrantGrpcClient.newBuilder(QDRANT_URL, 6334, true)
|
||||
.withApiKey(QDRANT_API_KEY)
|
||||
.build());
|
||||
```
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
```python
|
||||
from qdrant_client import QdrantClient, models
|
||||
|
||||
client = QdrantClient(
|
||||
url=QDRANT_URL,
|
||||
api_key=QDRANT_API_KEY,
|
||||
cloud_inference=True
|
||||
)
|
||||
```
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
```rust
|
||||
let client = Qdrant::from_url(QDRANT_URL)
|
||||
.api_key(QDRANT_API_KEY)
|
||||
.build()?;
|
||||
```
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
```typescript
|
||||
const client = new QdrantClient({
|
||||
url: QDRANT_URL,
|
||||
apiKey: QDRANT_API_KEY,
|
||||
});
|
||||
```
|
||||
+278
@@ -0,0 +1,278 @@
|
||||
```java
|
||||
import static io.qdrant.client.PointIdFactory.id;
|
||||
import static io.qdrant.client.QueryFactory.nearest;
|
||||
import static io.qdrant.client.ShardKeyFactory.shardKey;
|
||||
import static io.qdrant.client.ShardKeySelectorFactory.shardKeySelector;
|
||||
import static io.qdrant.client.ValueFactory.value;
|
||||
import static io.qdrant.client.VectorFactory.vector;
|
||||
import static io.qdrant.client.VectorsFactory.namedVectors;
|
||||
|
||||
import io.qdrant.client.QdrantClient;
|
||||
import io.qdrant.client.QdrantGrpcClient;
|
||||
import io.qdrant.client.grpc.Collections.CreateCollection;
|
||||
import io.qdrant.client.grpc.Collections.CreateShardKey;
|
||||
import io.qdrant.client.grpc.Collections.CreateShardKeyRequest;
|
||||
import io.qdrant.client.grpc.Collections.DeleteShardKey;
|
||||
import io.qdrant.client.grpc.Collections.DeleteShardKeyRequest;
|
||||
import io.qdrant.client.grpc.Collections.Distance;
|
||||
import io.qdrant.client.grpc.Collections.ShardingMethod;
|
||||
import io.qdrant.client.grpc.Collections.VectorParams;
|
||||
import io.qdrant.client.grpc.Collections.VectorParamsMap;
|
||||
import io.qdrant.client.grpc.Collections.VectorsConfig;
|
||||
import io.qdrant.client.grpc.Points.Document;
|
||||
import io.qdrant.client.grpc.Points.PointStruct;
|
||||
import io.qdrant.client.grpc.Points.QueryPoints;
|
||||
import io.qdrant.client.grpc.Points.ShardKeySelector;
|
||||
import io.qdrant.client.grpc.Points.UpsertPoints;
|
||||
import java.io.BufferedReader;
|
||||
import java.io.InputStreamReader;
|
||||
import java.net.URL;
|
||||
import java.time.LocalDate;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
static class CsvRow {
|
||||
final String text;
|
||||
final String datetime;
|
||||
CsvRow(String text, String datetime) { this.text = text; this.datetime = datetime; }
|
||||
}
|
||||
|
||||
static Stream<CsvRow> parseCSV(String url) throws Exception {
|
||||
Function<String, List<String>> parseCsvLine = line -> {
|
||||
List<String> fields = new ArrayList<>();
|
||||
boolean inQuotes = false;
|
||||
var sb = new StringBuilder();
|
||||
for (char c : line.toCharArray()) {
|
||||
if (c == '"') {
|
||||
inQuotes = !inQuotes;
|
||||
} else if (c == ',' && !inQuotes) {
|
||||
fields.add(sb.toString());
|
||||
sb.setLength(0);
|
||||
} else {
|
||||
sb.append(c);
|
||||
}
|
||||
}
|
||||
fields.add(sb.toString());
|
||||
return fields;
|
||||
};
|
||||
|
||||
var reader = new BufferedReader(new InputStreamReader(new URL(url).openStream()));
|
||||
String headerLine = reader.readLine();
|
||||
List<String> headers = List.of(headerLine.split(","));
|
||||
int textIdx = headers.indexOf("text");
|
||||
int datetimeIdx = headers.indexOf("datetime");
|
||||
|
||||
return reader.lines()
|
||||
.map(line -> {
|
||||
List<String> fields = parseCsvLine.apply(line);
|
||||
return new CsvRow(fields.get(textIdx), fields.get(datetimeIdx));
|
||||
})
|
||||
.onClose(() -> { try { reader.close(); } catch (Exception ignored) {} });
|
||||
}
|
||||
|
||||
QdrantClient client =
|
||||
new QdrantClient(
|
||||
QdrantGrpcClient.newBuilder(QDRANT_URL, 6334, true)
|
||||
.withApiKey(QDRANT_API_KEY)
|
||||
.build());
|
||||
|
||||
String collectionName = "my_collection";
|
||||
|
||||
if (client.collectionExistsAsync(collectionName).get()) {
|
||||
client.deleteCollectionAsync(collectionName).get();
|
||||
}
|
||||
|
||||
client.createCollectionAsync(
|
||||
CreateCollection.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.setVectorsConfig(VectorsConfig.newBuilder().setParamsMap(
|
||||
VectorParamsMap.newBuilder().putAllMap(Map.of(
|
||||
"dense_vector",
|
||||
VectorParams.newBuilder()
|
||||
.setSize(384)
|
||||
.setDistance(Distance.Cosine)
|
||||
.build()))))
|
||||
.setShardingMethod(ShardingMethod.Custom)
|
||||
.build()
|
||||
).get();
|
||||
|
||||
String csvUrl = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv";
|
||||
|
||||
// Retrieve a list of existing shard keys in the collection
|
||||
var shardKeyDescriptions = client.listShardKeysAsync(collectionName).get();
|
||||
Set<String> existingShardKeys = new HashSet<>();
|
||||
for (var desc : shardKeyDescriptions) {
|
||||
existingShardKeys.add(desc.getKey().getKeyword());
|
||||
}
|
||||
|
||||
String denseModel = "sentence-transformers/all-MiniLM-L6-v2";
|
||||
int batchSize = 100;
|
||||
String currentDate = null;
|
||||
List<PointStruct> buffer = new ArrayList<>();
|
||||
|
||||
try (var stream = parseCSV(csvUrl)) {
|
||||
for (var row : (Iterable<CsvRow>) stream::iterator) {
|
||||
String text = row.text;
|
||||
String datetime = row.datetime;
|
||||
String shardDate = datetime.substring(0, 10); // Extract YYYY-MM-DD
|
||||
|
||||
if (!shardDate.equals(currentDate)) {
|
||||
// Flush buffer for the previous date before switching
|
||||
if (!buffer.isEmpty()) {
|
||||
client.upsertAsync(
|
||||
UpsertPoints.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.addAllPoints(buffer)
|
||||
.setShardKeySelector(shardKeySelector(currentDate))
|
||||
.build()
|
||||
).get();
|
||||
buffer.clear();
|
||||
}
|
||||
|
||||
// Create shard for the new date if it doesn't exist yet
|
||||
if (!existingShardKeys.contains(shardDate)) {
|
||||
client.createShardKeyAsync(
|
||||
CreateShardKeyRequest.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.setRequest(CreateShardKey.newBuilder()
|
||||
.setShardKey(shardKey(shardDate))
|
||||
.build())
|
||||
.build()
|
||||
).get();
|
||||
existingShardKeys.add(shardDate);
|
||||
}
|
||||
|
||||
currentDate = shardDate;
|
||||
}
|
||||
|
||||
// Add point to buffer
|
||||
buffer.add(
|
||||
PointStruct.newBuilder()
|
||||
.setId(id(UUID.randomUUID()))
|
||||
.setVectors(namedVectors(Map.of(
|
||||
"dense_vector",
|
||||
vector(Document.newBuilder()
|
||||
.setText(text)
|
||||
.setModel(denseModel)
|
||||
.build()))))
|
||||
.putAllPayload(Map.of("text", value(text), "datetime", value(datetime)))
|
||||
.build());
|
||||
|
||||
// Flush batch if buffer size exceeds batch size
|
||||
if (buffer.size() >= batchSize) {
|
||||
client.upsertAsync(
|
||||
UpsertPoints.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.addAllPoints(buffer)
|
||||
.setShardKeySelector(shardKeySelector(currentDate))
|
||||
.build()
|
||||
).get();
|
||||
buffer.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Flush remaining partial batch
|
||||
if (!buffer.isEmpty()) {
|
||||
client.upsertAsync(
|
||||
UpsertPoints.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.addAllPoints(buffer)
|
||||
.setShardKeySelector(shardKeySelector(currentDate))
|
||||
.build()
|
||||
).get();
|
||||
}
|
||||
|
||||
String queryText = "coffee";
|
||||
|
||||
var result = client.queryAsync(
|
||||
QueryPoints.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.setQuery(nearest(Document.newBuilder().setText(queryText).setModel(denseModel).build()))
|
||||
.setUsing("dense_vector")
|
||||
.setLimit(5)
|
||||
.setShardKeySelector(shardKeySelector("2026-04-07"))
|
||||
.build()
|
||||
).get();
|
||||
|
||||
for (var hit : result) {
|
||||
System.out.println(hit);
|
||||
}
|
||||
|
||||
result = client.queryAsync(
|
||||
QueryPoints.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.setQuery(nearest(Document.newBuilder().setText(queryText).setModel(denseModel).build()))
|
||||
.setUsing("dense_vector")
|
||||
.setLimit(5)
|
||||
.setShardKeySelector(ShardKeySelector.newBuilder()
|
||||
.addShardKeys(shardKey("2026-04-06"))
|
||||
.addShardKeys(shardKey("2026-04-07"))
|
||||
.build())
|
||||
.build()
|
||||
).get();
|
||||
|
||||
for (var hit : result) {
|
||||
System.out.println(hit);
|
||||
}
|
||||
|
||||
result = client.queryAsync(
|
||||
QueryPoints.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.setQuery(nearest(Document.newBuilder().setText(queryText).setModel(denseModel).build()))
|
||||
.setUsing("dense_vector")
|
||||
.setLimit(5)
|
||||
.build()
|
||||
).get();
|
||||
|
||||
for (var hit : result) {
|
||||
System.out.println(hit);
|
||||
}
|
||||
|
||||
String today = "2026-04-08";
|
||||
String oldestShardKey = LocalDate.parse(today).minusDays(7).toString();
|
||||
|
||||
client.createShardKeyAsync(
|
||||
CreateShardKeyRequest.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.setRequest(CreateShardKey.newBuilder()
|
||||
.setShardKey(shardKey(today))
|
||||
.build())
|
||||
.build()
|
||||
).get();
|
||||
|
||||
client.deleteShardKeyAsync(
|
||||
DeleteShardKeyRequest.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.setRequest(DeleteShardKey.newBuilder()
|
||||
.setShardKey(shardKey(oldestShardKey))
|
||||
.build())
|
||||
.build()
|
||||
).get();
|
||||
|
||||
client.upsertAsync(
|
||||
UpsertPoints.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.addAllPoints(List.of(
|
||||
PointStruct.newBuilder()
|
||||
.setId(id(UUID.randomUUID()))
|
||||
.setVectors(namedVectors(Map.of(
|
||||
"dense_vector",
|
||||
vector(Document.newBuilder()
|
||||
.setText("The best way to start a Wednesday is with a cup of coffee")
|
||||
.setModel(denseModel)
|
||||
.build()))))
|
||||
.putAllPayload(Map.of(
|
||||
"text", value("The best way to start a Wednesday is with a cup of coffee"),
|
||||
"datetime", value("2026-04-08T07:57:47")))
|
||||
.build()))
|
||||
.setShardKeySelector(shardKeySelector(today))
|
||||
.build()
|
||||
).get();
|
||||
```
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
```csharp
|
||||
async IAsyncEnumerable<(string text, string datetime)> ParseCsv(string url)
|
||||
{
|
||||
using var httpClient = new HttpClient();
|
||||
using var stream = await httpClient.GetStreamAsync(url);
|
||||
using var parser = new TextFieldParser(new StreamReader(stream));
|
||||
parser.TextFieldType = Microsoft.VisualBasic.FileIO.FieldType.Delimited;
|
||||
parser.SetDelimiters(",");
|
||||
string[]? headers = parser.ReadFields();
|
||||
int textIdx = Array.IndexOf(headers!, "text");
|
||||
int datetimeIdx = Array.IndexOf(headers!, "datetime");
|
||||
while (!parser.EndOfData)
|
||||
{
|
||||
var fields = parser.ReadFields()!;
|
||||
yield return (fields[textIdx], fields[datetimeIdx]);
|
||||
}
|
||||
}
|
||||
```
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
```go
|
||||
type CSVRow struct {
|
||||
Text string
|
||||
Datetime string
|
||||
}
|
||||
|
||||
func parseCSV(url string, fn func(CSVRow)) error {
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
csvReader := csv.NewReader(resp.Body)
|
||||
headers, err := csvReader.Read()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
textIdx, datetimeIdx := -1, -1
|
||||
for i, h := range headers {
|
||||
switch h {
|
||||
case "text":
|
||||
textIdx = i
|
||||
case "datetime":
|
||||
datetimeIdx = i
|
||||
}
|
||||
}
|
||||
|
||||
for {
|
||||
row, err := csvReader.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fn(CSVRow{Text: row[textIdx], Datetime: row[datetimeIdx]})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
```
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
```java
|
||||
static class CsvRow {
|
||||
final String text;
|
||||
final String datetime;
|
||||
CsvRow(String text, String datetime) { this.text = text; this.datetime = datetime; }
|
||||
}
|
||||
|
||||
static Stream<CsvRow> parseCSV(String url) throws Exception {
|
||||
Function<String, List<String>> parseCsvLine = line -> {
|
||||
List<String> fields = new ArrayList<>();
|
||||
boolean inQuotes = false;
|
||||
var sb = new StringBuilder();
|
||||
for (char c : line.toCharArray()) {
|
||||
if (c == '"') {
|
||||
inQuotes = !inQuotes;
|
||||
} else if (c == ',' && !inQuotes) {
|
||||
fields.add(sb.toString());
|
||||
sb.setLength(0);
|
||||
} else {
|
||||
sb.append(c);
|
||||
}
|
||||
}
|
||||
fields.add(sb.toString());
|
||||
return fields;
|
||||
};
|
||||
|
||||
var reader = new BufferedReader(new InputStreamReader(new URL(url).openStream()));
|
||||
String headerLine = reader.readLine();
|
||||
List<String> headers = List.of(headerLine.split(","));
|
||||
int textIdx = headers.indexOf("text");
|
||||
int datetimeIdx = headers.indexOf("datetime");
|
||||
|
||||
return reader.lines()
|
||||
.map(line -> {
|
||||
List<String> fields = parseCsvLine.apply(line);
|
||||
return new CsvRow(fields.get(textIdx), fields.get(datetimeIdx));
|
||||
})
|
||||
.onClose(() -> { try { reader.close(); } catch (Exception ignored) {} });
|
||||
}
|
||||
```
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
```python
|
||||
import csv
|
||||
import urllib.request
|
||||
|
||||
def parse_csv(url):
|
||||
with urllib.request.urlopen(url) as response:
|
||||
reader = csv.DictReader(line.decode('utf-8') for line in response)
|
||||
yield from reader
|
||||
```
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
```rust
|
||||
struct CsvRow {
|
||||
text: String,
|
||||
datetime: String,
|
||||
}
|
||||
|
||||
fn parse_csv(url: &str) -> anyhow::Result<impl Iterator<Item = anyhow::Result<CsvRow>>> {
|
||||
let reader = ureq::get(url).call()?.into_body().into_reader();
|
||||
let mut rdr = csv::Reader::from_reader(reader);
|
||||
let headers = rdr.headers()?.clone();
|
||||
let text_idx = headers.iter().position(|h| h == "text").unwrap();
|
||||
let datetime_idx = headers.iter().position(|h| h == "datetime").unwrap();
|
||||
let iter = rdr.into_records().map(move |result| {
|
||||
let record = result?;
|
||||
Ok(CsvRow {
|
||||
text: record[text_idx].to_string(),
|
||||
datetime: record[datetime_idx].to_string(),
|
||||
})
|
||||
});
|
||||
Ok(iter)
|
||||
}
|
||||
```
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
```typescript
|
||||
function parseCsvLine(line: string): string[] {
|
||||
const fields: string[] = [];
|
||||
let i = 0;
|
||||
while (i < line.length) {
|
||||
if (line[i] === '"') {
|
||||
i++;
|
||||
let field = "";
|
||||
while (i < line.length) {
|
||||
if (line[i] === '"' && line[i + 1] === '"') { field += '"'; i += 2; }
|
||||
else if (line[i] === '"') { i++; break; }
|
||||
else { field += line[i++]; }
|
||||
}
|
||||
fields.push(field);
|
||||
if (line[i] === ",") i++;
|
||||
} else {
|
||||
const start = i;
|
||||
while (i < line.length && line[i] !== ",") i++;
|
||||
fields.push(line.slice(start, i));
|
||||
if (i < line.length) i++;
|
||||
}
|
||||
}
|
||||
return fields;
|
||||
}
|
||||
|
||||
async function* parseCSV(url: string): AsyncGenerator<{ text: string; datetime: string }> {
|
||||
const response = await fetch(url);
|
||||
const reader = response.body!.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let remainder = "";
|
||||
let headers: string[] | null = null;
|
||||
let textIdx = -1;
|
||||
let datetimeIdx = -1;
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
const chunk = done ? "" : decoder.decode(value, { stream: true });
|
||||
const lines = (remainder + chunk).split("\n");
|
||||
remainder = done ? "" : lines.pop()!;
|
||||
|
||||
for (const line of lines) {
|
||||
if (!line.trim()) continue;
|
||||
if (headers === null) {
|
||||
headers = line.split(",");
|
||||
textIdx = headers.indexOf("text");
|
||||
datetimeIdx = headers.indexOf("datetime");
|
||||
continue;
|
||||
}
|
||||
const fields = parseCsvLine(line);
|
||||
yield { text: fields[textIdx], datetime: fields[datetimeIdx] };
|
||||
}
|
||||
|
||||
if (done) break;
|
||||
}
|
||||
}
|
||||
```
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
```csharp
|
||||
string today = "2026-04-08";
|
||||
string oldestShardKey = DateOnly.ParseExact(today, "yyyy-MM-dd")
|
||||
.AddDays(-7)
|
||||
.ToString("yyyy-MM-dd");
|
||||
|
||||
await client.CreateShardKeyAsync(
|
||||
collectionName,
|
||||
new CreateShardKey { ShardKey = new ShardKey { Keyword = today } }
|
||||
);
|
||||
await client.DeleteShardKeyAsync(
|
||||
collectionName,
|
||||
new DeleteShardKey { ShardKey = new ShardKey { Keyword = oldestShardKey } }
|
||||
);
|
||||
```
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
```go
|
||||
today := "2026-04-08"
|
||||
t, _ := time.Parse("2006-01-02", today)
|
||||
oldestShardKey := t.AddDate(0, 0, -7).Format("2006-01-02")
|
||||
|
||||
client.CreateShardKey(context.Background(), collectionName, &qdrant.CreateShardKey{
|
||||
ShardKey: qdrant.NewShardKey(today),
|
||||
})
|
||||
client.DeleteShardKey(context.Background(), collectionName, &qdrant.DeleteShardKey{
|
||||
ShardKey: qdrant.NewShardKey(oldestShardKey),
|
||||
})
|
||||
```
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
```java
|
||||
String today = "2026-04-08";
|
||||
String oldestShardKey = LocalDate.parse(today).minusDays(7).toString();
|
||||
|
||||
client.createShardKeyAsync(
|
||||
CreateShardKeyRequest.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.setRequest(CreateShardKey.newBuilder()
|
||||
.setShardKey(shardKey(today))
|
||||
.build())
|
||||
.build()
|
||||
).get();
|
||||
|
||||
client.deleteShardKeyAsync(
|
||||
DeleteShardKeyRequest.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.setRequest(DeleteShardKey.newBuilder()
|
||||
.setShardKey(shardKey(oldestShardKey))
|
||||
.build())
|
||||
.build()
|
||||
).get();
|
||||
```
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
```python
|
||||
from datetime import date, timedelta
|
||||
|
||||
today = "2026-04-08"
|
||||
oldest_shard_key = (date.fromisoformat(today) - timedelta(days=7)).isoformat()
|
||||
|
||||
client.create_shard_key(collection_name, today)
|
||||
client.delete_shard_key(collection_name, oldest_shard_key)
|
||||
```
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
```rust
|
||||
let today = "2026-04-08";
|
||||
let oldest_shard_key = (NaiveDate::parse_from_str(today, "%Y-%m-%d")?
|
||||
- chrono::Duration::days(7))
|
||||
.to_string();
|
||||
|
||||
client
|
||||
.create_shard_key(
|
||||
CreateShardKeyRequestBuilder::new(collection_name)
|
||||
.request(CreateShardKeyBuilder::default().shard_key(today.to_string())),
|
||||
)
|
||||
.await?;
|
||||
|
||||
client
|
||||
.delete_shard_key(
|
||||
DeleteShardKeyRequestBuilder::new(collection_name)
|
||||
.key(shard_key::Key::Keyword(oldest_shard_key)),
|
||||
)
|
||||
.await?;
|
||||
```
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
```typescript
|
||||
const today = "2026-04-08";
|
||||
const oldestDate = new Date(today);
|
||||
oldestDate.setDate(oldestDate.getDate() - 7);
|
||||
const oldestShardKey = oldestDate.toISOString().slice(0, 10);
|
||||
|
||||
await client.createShardKey(collectionName, { shard_key: today });
|
||||
await client.deleteShardKey(collectionName, { shard_key: oldestShardKey });
|
||||
```
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
```python
|
||||
from qdrant_client import QdrantClient, models
|
||||
|
||||
client = QdrantClient(
|
||||
url=QDRANT_URL,
|
||||
api_key=QDRANT_API_KEY,
|
||||
cloud_inference=True
|
||||
)
|
||||
|
||||
from qdrant_client import models
|
||||
|
||||
collection_name = "my_collection"
|
||||
|
||||
if client.collection_exists(collection_name=collection_name):
|
||||
client.delete_collection(collection_name=collection_name)
|
||||
|
||||
client.create_collection(
|
||||
collection_name=collection_name,
|
||||
vectors_config={
|
||||
"dense_vector": models.VectorParams(
|
||||
size=384, distance=models.Distance.COSINE
|
||||
)
|
||||
},
|
||||
sharding_method=models.ShardingMethod.CUSTOM
|
||||
)
|
||||
|
||||
import csv
|
||||
import urllib.request
|
||||
|
||||
def parse_csv(url):
|
||||
with urllib.request.urlopen(url) as response:
|
||||
reader = csv.DictReader(line.decode('utf-8') for line in response)
|
||||
yield from reader
|
||||
|
||||
from qdrant_client.http.models import PointStruct, Document
|
||||
import uuid
|
||||
|
||||
csv_url = 'https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv'
|
||||
|
||||
# Retrieve a list of existing shard keys in the collection
|
||||
existing_shard_keys = list(client.list_shard_keys(collection_name=collection_name).shard_keys)
|
||||
|
||||
dense_model = "sentence-transformers/all-MiniLM-L6-v2"
|
||||
batch_size = 100
|
||||
current_date = None
|
||||
buffer: list[PointStruct] = []
|
||||
|
||||
for row in parse_csv(csv_url):
|
||||
shard_date = row['datetime'][:10] # Extract YYYY-MM-DD
|
||||
|
||||
if shard_date != current_date:
|
||||
# Flush buffer for the previous date before switching
|
||||
if buffer:
|
||||
client.upload_points(
|
||||
collection_name=collection_name,
|
||||
points=buffer,
|
||||
shard_key_selector=current_date,
|
||||
)
|
||||
buffer = []
|
||||
|
||||
# Create shard for the new date if it doesn't exist yet
|
||||
if shard_date not in existing_shard_keys:
|
||||
client.create_shard_key(collection_name, shard_date)
|
||||
existing_shard_keys.append(shard_date)
|
||||
|
||||
current_date = shard_date
|
||||
|
||||
# Add point to buffer
|
||||
buffer.append(PointStruct(
|
||||
id=uuid.uuid4().hex,
|
||||
payload={"text": row['text'], "datetime": row['datetime']},
|
||||
vector={"dense_vector": Document(text=row["text"], model=dense_model)}
|
||||
))
|
||||
|
||||
# Flush batch if buffer size exceeds batch size
|
||||
if len(buffer) >= batch_size:
|
||||
client.upload_points(
|
||||
collection_name=collection_name,
|
||||
points=buffer,
|
||||
shard_key_selector=current_date,
|
||||
)
|
||||
buffer = []
|
||||
|
||||
# Flush remaining partial batch
|
||||
if buffer:
|
||||
client.upload_points(
|
||||
collection_name=collection_name,
|
||||
points=buffer,
|
||||
shard_key_selector=current_date,
|
||||
)
|
||||
|
||||
query_text = "coffee"
|
||||
|
||||
resp = client.query_points(
|
||||
collection_name=collection_name,
|
||||
query=Document(text=query_text, model=dense_model),
|
||||
using="dense_vector",
|
||||
limit=5,
|
||||
shard_key_selector="2026-04-07"
|
||||
)
|
||||
print(resp)
|
||||
|
||||
resp = client.query_points(
|
||||
collection_name=collection_name,
|
||||
query=Document(text=query_text, model=dense_model),
|
||||
using="dense_vector",
|
||||
limit=5,
|
||||
shard_key_selector=["2026-04-06","2026-04-07"]
|
||||
)
|
||||
print(resp)
|
||||
|
||||
resp = client.query_points(
|
||||
collection_name=collection_name,
|
||||
query=Document(text=query_text, model=dense_model),
|
||||
using="dense_vector",
|
||||
limit=5,
|
||||
)
|
||||
print(resp)
|
||||
|
||||
from datetime import date, timedelta
|
||||
|
||||
today = "2026-04-08"
|
||||
oldest_shard_key = (date.fromisoformat(today) - timedelta(days=7)).isoformat()
|
||||
|
||||
client.create_shard_key(collection_name, today)
|
||||
client.delete_shard_key(collection_name, oldest_shard_key)
|
||||
|
||||
client.upsert(
|
||||
collection_name=collection_name,
|
||||
points=[PointStruct(
|
||||
id=uuid.uuid4().hex,
|
||||
payload={"text": "The best way to start a Wednesday is with a cup of coffee", "datetime": "2026-04-08T07:57:47"},
|
||||
vector={
|
||||
"dense_vector": Document(text="The best way to start a Wednesday is with a cup of coffee", model=dense_model)
|
||||
})],
|
||||
shard_key_selector=today
|
||||
)
|
||||
```
|
||||
+235
@@ -0,0 +1,235 @@
|
||||
```rust
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use chrono::NaiveDate;
|
||||
use qdrant_client::Qdrant;
|
||||
use qdrant_client::qdrant::{
|
||||
CreateCollectionBuilder, CreateShardKeyBuilder, CreateShardKeyRequestBuilder,
|
||||
DeleteShardKeyRequestBuilder, Distance, Document, DocumentBuilder,
|
||||
PointStruct, Query, QueryPointsBuilder, ShardKeySelector, ShardingMethod,
|
||||
UpsertPointsBuilder, VectorParamsBuilder, VectorsConfigBuilder, shard_key,
|
||||
};
|
||||
|
||||
let client = Qdrant::from_url(QDRANT_URL)
|
||||
.api_key(QDRANT_API_KEY)
|
||||
.build()?;
|
||||
|
||||
let collection_name = "my_collection";
|
||||
|
||||
if client.collection_exists(collection_name).await? {
|
||||
client.delete_collection(collection_name).await?;
|
||||
}
|
||||
|
||||
let mut vectors_config = VectorsConfigBuilder::default();
|
||||
vectors_config.add_named_vector_params(
|
||||
"dense_vector",
|
||||
VectorParamsBuilder::new(384, Distance::Cosine),
|
||||
);
|
||||
|
||||
client
|
||||
.create_collection(
|
||||
CreateCollectionBuilder::new(collection_name)
|
||||
.vectors_config(vectors_config)
|
||||
.sharding_method(ShardingMethod::Custom.into()),
|
||||
)
|
||||
.await?;
|
||||
|
||||
struct CsvRow {
|
||||
text: String,
|
||||
datetime: String,
|
||||
}
|
||||
|
||||
fn parse_csv(url: &str) -> anyhow::Result<impl Iterator<Item = anyhow::Result<CsvRow>>> {
|
||||
let reader = ureq::get(url).call()?.into_body().into_reader();
|
||||
let mut rdr = csv::Reader::from_reader(reader);
|
||||
let headers = rdr.headers()?.clone();
|
||||
let text_idx = headers.iter().position(|h| h == "text").unwrap();
|
||||
let datetime_idx = headers.iter().position(|h| h == "datetime").unwrap();
|
||||
let iter = rdr.into_records().map(move |result| {
|
||||
let record = result?;
|
||||
Ok(CsvRow {
|
||||
text: record[text_idx].to_string(),
|
||||
datetime: record[datetime_idx].to_string(),
|
||||
})
|
||||
});
|
||||
Ok(iter)
|
||||
}
|
||||
|
||||
let csv_url = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv";
|
||||
|
||||
// Retrieve a list of existing shard keys in the collection
|
||||
let response = client.list_shard_keys(collection_name).await?;
|
||||
let mut existing_shard_keys: HashSet<String> = response
|
||||
.shard_keys
|
||||
.into_iter()
|
||||
.filter_map(|d| {
|
||||
d.key?.key.and_then(|k| match k {
|
||||
shard_key::Key::Keyword(s) => Some(s),
|
||||
_ => None,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
let dense_model = "sentence-transformers/all-MiniLM-L6-v2";
|
||||
let batch_size = 100;
|
||||
let mut current_date = String::new();
|
||||
let mut buffer: Vec<PointStruct> = Vec::new();
|
||||
|
||||
for row in parse_csv(csv_url)? {
|
||||
let row = row?;
|
||||
let text = row.text;
|
||||
let datetime = row.datetime;
|
||||
let shard_date = datetime[..10].to_string(); // Extract YYYY-MM-DD
|
||||
|
||||
if shard_date != current_date {
|
||||
// Flush buffer for the previous date before switching
|
||||
if !buffer.is_empty() {
|
||||
client
|
||||
.upsert_points(
|
||||
UpsertPointsBuilder::new(
|
||||
collection_name,
|
||||
std::mem::take(&mut buffer),
|
||||
)
|
||||
.shard_key_selector(current_date.clone()),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
// Create shard for the new date if it doesn't exist yet
|
||||
if !existing_shard_keys.contains(&shard_date) {
|
||||
client
|
||||
.create_shard_key(
|
||||
CreateShardKeyRequestBuilder::new(collection_name).request(
|
||||
CreateShardKeyBuilder::default().shard_key(shard_date.clone()),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
existing_shard_keys.insert(shard_date.clone());
|
||||
}
|
||||
|
||||
current_date = shard_date;
|
||||
}
|
||||
|
||||
// Add point to buffer
|
||||
buffer.push(PointStruct::new(
|
||||
uuid::Uuid::new_v4().to_string(),
|
||||
HashMap::from([(
|
||||
"dense_vector".to_string(),
|
||||
DocumentBuilder::new(&text, dense_model).build(),
|
||||
)]),
|
||||
[("text", text.into()), ("datetime", datetime.into())],
|
||||
));
|
||||
|
||||
// Flush batch if buffer size exceeds batch size
|
||||
if buffer.len() >= batch_size {
|
||||
client
|
||||
.upsert_points(
|
||||
UpsertPointsBuilder::new(collection_name, std::mem::take(&mut buffer))
|
||||
.shard_key_selector(current_date.clone()),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
|
||||
// Flush remaining partial batch
|
||||
if !buffer.is_empty() {
|
||||
client
|
||||
.upsert_points(
|
||||
UpsertPointsBuilder::new(collection_name, buffer)
|
||||
.shard_key_selector(current_date.clone()),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
let query_text = "coffee";
|
||||
|
||||
let result = client
|
||||
.query(
|
||||
QueryPointsBuilder::new(collection_name)
|
||||
.query(Query::new_nearest(Document::new(query_text, dense_model)))
|
||||
.using("dense_vector")
|
||||
.limit(5)
|
||||
.shard_key_selector("2026-04-07".to_string()),
|
||||
)
|
||||
.await?;
|
||||
|
||||
for hit in result.result {
|
||||
println!("{:?}", hit);
|
||||
}
|
||||
|
||||
let result = client
|
||||
.query(
|
||||
QueryPointsBuilder::new(collection_name)
|
||||
.query(Query::new_nearest(Document::new(query_text, dense_model)))
|
||||
.using("dense_vector")
|
||||
.limit(5)
|
||||
.shard_key_selector(ShardKeySelector {
|
||||
shard_keys: vec![
|
||||
"2026-04-06".to_string().into(),
|
||||
"2026-04-07".to_string().into(),
|
||||
],
|
||||
fallback: None,
|
||||
}),
|
||||
)
|
||||
.await?;
|
||||
|
||||
for hit in result.result {
|
||||
println!("{:?}", hit);
|
||||
}
|
||||
|
||||
let result = client
|
||||
.query(
|
||||
QueryPointsBuilder::new(collection_name)
|
||||
.query(Query::new_nearest(Document::new(query_text, dense_model)))
|
||||
.using("dense_vector")
|
||||
.limit(5),
|
||||
)
|
||||
.await?;
|
||||
|
||||
for hit in result.result {
|
||||
println!("{:?}", hit);
|
||||
}
|
||||
|
||||
let today = "2026-04-08";
|
||||
let oldest_shard_key = (NaiveDate::parse_from_str(today, "%Y-%m-%d")?
|
||||
- chrono::Duration::days(7))
|
||||
.to_string();
|
||||
|
||||
client
|
||||
.create_shard_key(
|
||||
CreateShardKeyRequestBuilder::new(collection_name)
|
||||
.request(CreateShardKeyBuilder::default().shard_key(today.to_string())),
|
||||
)
|
||||
.await?;
|
||||
|
||||
client
|
||||
.delete_shard_key(
|
||||
DeleteShardKeyRequestBuilder::new(collection_name)
|
||||
.key(shard_key::Key::Keyword(oldest_shard_key)),
|
||||
)
|
||||
.await?;
|
||||
|
||||
client
|
||||
.upsert_points(
|
||||
UpsertPointsBuilder::new(
|
||||
collection_name,
|
||||
vec![PointStruct::new(
|
||||
uuid::Uuid::new_v4().to_string(),
|
||||
HashMap::from([(
|
||||
"dense_vector".to_string(),
|
||||
DocumentBuilder::new(
|
||||
"The best way to start a Wednesday is with a cup of coffee",
|
||||
dense_model,
|
||||
)
|
||||
.build(),
|
||||
)]),
|
||||
[
|
||||
("text", "The best way to start a Wednesday is with a cup of coffee".into()),
|
||||
("datetime", "2026-04-08T07:57:47".into()),
|
||||
],
|
||||
)],
|
||||
)
|
||||
.shard_key_selector(today.to_string()),
|
||||
)
|
||||
.await?;
|
||||
```
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
```csharp
|
||||
result = await client.QueryAsync(
|
||||
collectionName: collectionName,
|
||||
query: new Document { Text = queryText, Model = denseModel },
|
||||
usingVector: "dense_vector",
|
||||
limit: 5
|
||||
);
|
||||
|
||||
foreach (var hit in result)
|
||||
Console.WriteLine(hit);
|
||||
```
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
```go
|
||||
result, err = client.Query(context.Background(), &qdrant.QueryPoints{
|
||||
CollectionName: collectionName,
|
||||
Query: qdrant.NewQueryDocument(&qdrant.Document{Text: queryText, Model: denseModel}),
|
||||
Using: qdrant.PtrOf("dense_vector"),
|
||||
Limit: qdrant.PtrOf(uint64(5)),
|
||||
})
|
||||
|
||||
for _, hit := range result {
|
||||
fmt.Println(hit)
|
||||
}
|
||||
```
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
```java
|
||||
result = client.queryAsync(
|
||||
QueryPoints.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.setQuery(nearest(Document.newBuilder().setText(queryText).setModel(denseModel).build()))
|
||||
.setUsing("dense_vector")
|
||||
.setLimit(5)
|
||||
.build()
|
||||
).get();
|
||||
|
||||
for (var hit : result) {
|
||||
System.out.println(hit);
|
||||
}
|
||||
```
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
```python
|
||||
resp = client.query_points(
|
||||
collection_name=collection_name,
|
||||
query=Document(text=query_text, model=dense_model),
|
||||
using="dense_vector",
|
||||
limit=5,
|
||||
)
|
||||
print(resp)
|
||||
```
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
```rust
|
||||
let result = client
|
||||
.query(
|
||||
QueryPointsBuilder::new(collection_name)
|
||||
.query(Query::new_nearest(Document::new(query_text, dense_model)))
|
||||
.using("dense_vector")
|
||||
.limit(5),
|
||||
)
|
||||
.await?;
|
||||
|
||||
for hit in result.result {
|
||||
println!("{:?}", hit);
|
||||
}
|
||||
```
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
```typescript
|
||||
const allShardsResult = await client.query(collectionName, {
|
||||
query: { text: queryText, model: denseModel },
|
||||
using: "dense_vector",
|
||||
limit: 5,
|
||||
});
|
||||
|
||||
for (const hit of allShardsResult.points) {
|
||||
console.log(hit);
|
||||
}
|
||||
```
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
```csharp
|
||||
result = await client.QueryAsync(
|
||||
collectionName: collectionName,
|
||||
query: new Document { Text = queryText, Model = denseModel },
|
||||
usingVector: "dense_vector",
|
||||
limit: 5,
|
||||
shardKeySelector: new ShardKeySelector
|
||||
{
|
||||
ShardKeys = { new List<ShardKey> { "2026-04-06", "2026-04-07" } }
|
||||
}
|
||||
);
|
||||
|
||||
foreach (var hit in result)
|
||||
Console.WriteLine(hit);
|
||||
```
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
```go
|
||||
result, err = client.Query(context.Background(), &qdrant.QueryPoints{
|
||||
CollectionName: collectionName,
|
||||
Query: qdrant.NewQueryDocument(&qdrant.Document{Text: queryText, Model: denseModel}),
|
||||
Using: qdrant.PtrOf("dense_vector"),
|
||||
Limit: qdrant.PtrOf(uint64(5)),
|
||||
ShardKeySelector: &qdrant.ShardKeySelector{
|
||||
ShardKeys: []*qdrant.ShardKey{
|
||||
qdrant.NewShardKey("2026-04-06"),
|
||||
qdrant.NewShardKey("2026-04-07"),
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
for _, hit := range result {
|
||||
fmt.Println(hit)
|
||||
}
|
||||
```
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
```java
|
||||
result = client.queryAsync(
|
||||
QueryPoints.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.setQuery(nearest(Document.newBuilder().setText(queryText).setModel(denseModel).build()))
|
||||
.setUsing("dense_vector")
|
||||
.setLimit(5)
|
||||
.setShardKeySelector(ShardKeySelector.newBuilder()
|
||||
.addShardKeys(shardKey("2026-04-06"))
|
||||
.addShardKeys(shardKey("2026-04-07"))
|
||||
.build())
|
||||
.build()
|
||||
).get();
|
||||
|
||||
for (var hit : result) {
|
||||
System.out.println(hit);
|
||||
}
|
||||
```
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
```python
|
||||
resp = client.query_points(
|
||||
collection_name=collection_name,
|
||||
query=Document(text=query_text, model=dense_model),
|
||||
using="dense_vector",
|
||||
limit=5,
|
||||
shard_key_selector=["2026-04-06","2026-04-07"]
|
||||
)
|
||||
print(resp)
|
||||
```
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
```rust
|
||||
let result = client
|
||||
.query(
|
||||
QueryPointsBuilder::new(collection_name)
|
||||
.query(Query::new_nearest(Document::new(query_text, dense_model)))
|
||||
.using("dense_vector")
|
||||
.limit(5)
|
||||
.shard_key_selector(ShardKeySelector {
|
||||
shard_keys: vec![
|
||||
"2026-04-06".to_string().into(),
|
||||
"2026-04-07".to_string().into(),
|
||||
],
|
||||
fallback: None,
|
||||
}),
|
||||
)
|
||||
.await?;
|
||||
|
||||
for hit in result.result {
|
||||
println!("{:?}", hit);
|
||||
}
|
||||
```
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
```typescript
|
||||
const multiShardResult = await client.query(collectionName, {
|
||||
query: { text: queryText, model: denseModel },
|
||||
using: "dense_vector",
|
||||
limit: 5,
|
||||
shard_key: ["2026-04-06", "2026-04-07"],
|
||||
});
|
||||
|
||||
for (const hit of multiShardResult.points) {
|
||||
console.log(hit);
|
||||
}
|
||||
```
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
```csharp
|
||||
string queryText = "coffee";
|
||||
|
||||
var result = await client.QueryAsync(
|
||||
collectionName: collectionName,
|
||||
query: new Document { Text = queryText, Model = denseModel },
|
||||
usingVector: "dense_vector",
|
||||
limit: 5,
|
||||
shardKeySelector: new ShardKeySelector
|
||||
{
|
||||
ShardKeys = { new List<ShardKey> { "2026-04-07" } }
|
||||
}
|
||||
);
|
||||
|
||||
foreach (var hit in result)
|
||||
Console.WriteLine(hit);
|
||||
```
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
```go
|
||||
queryText := "coffee"
|
||||
|
||||
result, err := client.Query(context.Background(), &qdrant.QueryPoints{
|
||||
CollectionName: collectionName,
|
||||
Query: qdrant.NewQueryDocument(&qdrant.Document{Text: queryText, Model: denseModel}),
|
||||
Using: qdrant.PtrOf("dense_vector"),
|
||||
Limit: qdrant.PtrOf(uint64(5)),
|
||||
ShardKeySelector: &qdrant.ShardKeySelector{
|
||||
ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey("2026-04-07")},
|
||||
},
|
||||
})
|
||||
|
||||
for _, hit := range result {
|
||||
fmt.Println(hit)
|
||||
}
|
||||
```
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
```java
|
||||
String queryText = "coffee";
|
||||
|
||||
var result = client.queryAsync(
|
||||
QueryPoints.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.setQuery(nearest(Document.newBuilder().setText(queryText).setModel(denseModel).build()))
|
||||
.setUsing("dense_vector")
|
||||
.setLimit(5)
|
||||
.setShardKeySelector(shardKeySelector("2026-04-07"))
|
||||
.build()
|
||||
).get();
|
||||
|
||||
for (var hit : result) {
|
||||
System.out.println(hit);
|
||||
}
|
||||
```
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
```python
|
||||
query_text = "coffee"
|
||||
|
||||
resp = client.query_points(
|
||||
collection_name=collection_name,
|
||||
query=Document(text=query_text, model=dense_model),
|
||||
using="dense_vector",
|
||||
limit=5,
|
||||
shard_key_selector="2026-04-07"
|
||||
)
|
||||
print(resp)
|
||||
```
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
```rust
|
||||
let query_text = "coffee";
|
||||
|
||||
let result = client
|
||||
.query(
|
||||
QueryPointsBuilder::new(collection_name)
|
||||
.query(Query::new_nearest(Document::new(query_text, dense_model)))
|
||||
.using("dense_vector")
|
||||
.limit(5)
|
||||
.shard_key_selector("2026-04-07".to_string()),
|
||||
)
|
||||
.await?;
|
||||
|
||||
for hit in result.result {
|
||||
println!("{:?}", hit);
|
||||
}
|
||||
```
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
```typescript
|
||||
const queryText = "coffee";
|
||||
|
||||
const singleShardResult = await client.query(collectionName, {
|
||||
query: { text: queryText, model: denseModel },
|
||||
using: "dense_vector",
|
||||
limit: 5,
|
||||
shard_key: "2026-04-07",
|
||||
});
|
||||
|
||||
for (const hit of singleShardResult.points) {
|
||||
console.log(hit);
|
||||
}
|
||||
```
|
||||
+189
@@ -0,0 +1,189 @@
|
||||
```typescript
|
||||
import { QdrantClient } from "@qdrant/js-client-rest";
|
||||
|
||||
const client = new QdrantClient({
|
||||
url: QDRANT_URL,
|
||||
apiKey: QDRANT_API_KEY,
|
||||
});
|
||||
|
||||
const collectionName = "my_collection";
|
||||
|
||||
if (await client.collectionExists(collectionName)) {
|
||||
await client.deleteCollection(collectionName);
|
||||
}
|
||||
|
||||
await client.createCollection(collectionName, {
|
||||
vectors: {
|
||||
dense_vector: {
|
||||
size: 384,
|
||||
distance: "Cosine",
|
||||
},
|
||||
},
|
||||
sharding_method: "custom",
|
||||
});
|
||||
|
||||
function parseCsvLine(line: string): string[] {
|
||||
const fields: string[] = [];
|
||||
let i = 0;
|
||||
while (i < line.length) {
|
||||
if (line[i] === '"') {
|
||||
i++;
|
||||
let field = "";
|
||||
while (i < line.length) {
|
||||
if (line[i] === '"' && line[i + 1] === '"') { field += '"'; i += 2; }
|
||||
else if (line[i] === '"') { i++; break; }
|
||||
else { field += line[i++]; }
|
||||
}
|
||||
fields.push(field);
|
||||
if (line[i] === ",") i++;
|
||||
} else {
|
||||
const start = i;
|
||||
while (i < line.length && line[i] !== ",") i++;
|
||||
fields.push(line.slice(start, i));
|
||||
if (i < line.length) i++;
|
||||
}
|
||||
}
|
||||
return fields;
|
||||
}
|
||||
|
||||
async function* parseCSV(url: string): AsyncGenerator<{ text: string; datetime: string }> {
|
||||
const response = await fetch(url);
|
||||
const reader = response.body!.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let remainder = "";
|
||||
let headers: string[] | null = null;
|
||||
let textIdx = -1;
|
||||
let datetimeIdx = -1;
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
const chunk = done ? "" : decoder.decode(value, { stream: true });
|
||||
const lines = (remainder + chunk).split("\n");
|
||||
remainder = done ? "" : lines.pop()!;
|
||||
|
||||
for (const line of lines) {
|
||||
if (!line.trim()) continue;
|
||||
if (headers === null) {
|
||||
headers = line.split(",");
|
||||
textIdx = headers.indexOf("text");
|
||||
datetimeIdx = headers.indexOf("datetime");
|
||||
continue;
|
||||
}
|
||||
const fields = parseCsvLine(line);
|
||||
yield { text: fields[textIdx], datetime: fields[datetimeIdx] };
|
||||
}
|
||||
|
||||
if (done) break;
|
||||
}
|
||||
}
|
||||
|
||||
const csvUrl = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv";
|
||||
|
||||
// Retrieve a list of existing shard keys in the collection
|
||||
const shardKeysResult = await client.listShardKeys(collectionName);
|
||||
const existingShardKeys = new Set((shardKeysResult.shard_keys ?? []).map((d) => String(d.key)));
|
||||
|
||||
const denseModel = "sentence-transformers/all-MiniLM-L6-v2";
|
||||
const batchSize = 100;
|
||||
let currentDate = "";
|
||||
let buffer: Extract<Parameters<typeof client.upsert>[1], { points: unknown }>['points'] = [];
|
||||
|
||||
for await (const { text: postText, datetime } of parseCSV(csvUrl)) {
|
||||
const shardDate = datetime.slice(0, 10); // Extract YYYY-MM-DD
|
||||
|
||||
if (shardDate !== currentDate) {
|
||||
// Flush buffer for the previous date before switching
|
||||
if (buffer.length > 0) {
|
||||
await client.upsert(collectionName, { points: buffer, shard_key: currentDate });
|
||||
buffer = [];
|
||||
}
|
||||
|
||||
// Create shard for the new date if it doesn't exist yet
|
||||
if (!existingShardKeys.has(shardDate)) {
|
||||
await client.createShardKey(collectionName, { shard_key: shardDate });
|
||||
existingShardKeys.add(shardDate);
|
||||
}
|
||||
|
||||
currentDate = shardDate;
|
||||
}
|
||||
|
||||
// Add point to buffer
|
||||
buffer.push({
|
||||
id: crypto.randomUUID(),
|
||||
vector: { dense_vector: { text: postText, model: denseModel } },
|
||||
payload: { text: postText, datetime },
|
||||
});
|
||||
|
||||
// Flush batch if buffer size exceeds batch size
|
||||
if (buffer.length >= batchSize) {
|
||||
await client.upsert(collectionName, { points: buffer, shard_key: currentDate });
|
||||
buffer = [];
|
||||
}
|
||||
}
|
||||
|
||||
// Flush remaining partial batch
|
||||
if (buffer.length > 0) {
|
||||
await client.upsert(collectionName, { points: buffer, shard_key: currentDate });
|
||||
}
|
||||
|
||||
const queryText = "coffee";
|
||||
|
||||
const singleShardResult = await client.query(collectionName, {
|
||||
query: { text: queryText, model: denseModel },
|
||||
using: "dense_vector",
|
||||
limit: 5,
|
||||
shard_key: "2026-04-07",
|
||||
});
|
||||
|
||||
for (const hit of singleShardResult.points) {
|
||||
console.log(hit);
|
||||
}
|
||||
|
||||
const multiShardResult = await client.query(collectionName, {
|
||||
query: { text: queryText, model: denseModel },
|
||||
using: "dense_vector",
|
||||
limit: 5,
|
||||
shard_key: ["2026-04-06", "2026-04-07"],
|
||||
});
|
||||
|
||||
for (const hit of multiShardResult.points) {
|
||||
console.log(hit);
|
||||
}
|
||||
|
||||
const allShardsResult = await client.query(collectionName, {
|
||||
query: { text: queryText, model: denseModel },
|
||||
using: "dense_vector",
|
||||
limit: 5,
|
||||
});
|
||||
|
||||
for (const hit of allShardsResult.points) {
|
||||
console.log(hit);
|
||||
}
|
||||
|
||||
const today = "2026-04-08";
|
||||
const oldestDate = new Date(today);
|
||||
oldestDate.setDate(oldestDate.getDate() - 7);
|
||||
const oldestShardKey = oldestDate.toISOString().slice(0, 10);
|
||||
|
||||
await client.createShardKey(collectionName, { shard_key: today });
|
||||
await client.deleteShardKey(collectionName, { shard_key: oldestShardKey });
|
||||
|
||||
await client.upsert(collectionName, {
|
||||
points: [
|
||||
{
|
||||
id: crypto.randomUUID(),
|
||||
vector: {
|
||||
dense_vector: {
|
||||
text: "The best way to start a Wednesday is with a cup of coffee",
|
||||
model: denseModel,
|
||||
},
|
||||
},
|
||||
payload: {
|
||||
text: "The best way to start a Wednesday is with a cup of coffee",
|
||||
datetime: "2026-04-08T07:57:47",
|
||||
},
|
||||
},
|
||||
],
|
||||
shard_key: today,
|
||||
});
|
||||
```
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
```csharp
|
||||
string csvUrl = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv";
|
||||
|
||||
// Retrieve a list of existing shard keys in the collection
|
||||
var existingShardKeys = (await client.ListShardKeysAsync(collectionName))
|
||||
.Select(sk => sk.Key.Keyword)
|
||||
.ToHashSet();
|
||||
|
||||
string denseModel = "sentence-transformers/all-MiniLM-L6-v2";
|
||||
int batchSize = 100;
|
||||
string? currentDate = null;
|
||||
var buffer = new List<PointStruct>();
|
||||
|
||||
await foreach (var (text, datetime) in ParseCsv(csvUrl))
|
||||
{
|
||||
string shardDate = datetime[..10]; // Extract YYYY-MM-DD
|
||||
|
||||
if (shardDate != currentDate)
|
||||
{
|
||||
// Flush buffer for the previous date before switching
|
||||
if (buffer.Count > 0)
|
||||
{
|
||||
await client.UpsertAsync(
|
||||
collectionName: collectionName,
|
||||
points: buffer,
|
||||
shardKeySelector: new ShardKeySelector
|
||||
{
|
||||
ShardKeys = { new List<ShardKey> { currentDate! } }
|
||||
}
|
||||
);
|
||||
buffer.Clear();
|
||||
}
|
||||
|
||||
// Create shard for the new date if it doesn't exist yet
|
||||
if (!existingShardKeys.Contains(shardDate))
|
||||
{
|
||||
await client.CreateShardKeyAsync(
|
||||
collectionName,
|
||||
new CreateShardKey { ShardKey = new ShardKey { Keyword = shardDate } }
|
||||
);
|
||||
existingShardKeys.Add(shardDate);
|
||||
}
|
||||
|
||||
currentDate = shardDate;
|
||||
}
|
||||
|
||||
// Add point to buffer
|
||||
buffer.Add(new PointStruct
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Vectors = new Dictionary<string, Vector>
|
||||
{
|
||||
["dense_vector"] = new Document { Text = text, Model = denseModel }
|
||||
},
|
||||
Payload = { ["text"] = text, ["datetime"] = datetime }
|
||||
});
|
||||
|
||||
// Flush batch if buffer size exceeds batch size
|
||||
if (buffer.Count >= batchSize)
|
||||
{
|
||||
await client.UpsertAsync(
|
||||
collectionName: collectionName,
|
||||
points: buffer,
|
||||
shardKeySelector: new ShardKeySelector
|
||||
{
|
||||
ShardKeys = { new List<ShardKey> { currentDate! } }
|
||||
}
|
||||
);
|
||||
buffer.Clear();
|
||||
}
|
||||
}
|
||||
|
||||
// Flush remaining partial batch
|
||||
if (buffer.Count > 0)
|
||||
{
|
||||
await client.UpsertAsync(
|
||||
collectionName: collectionName,
|
||||
points: buffer,
|
||||
shardKeySelector: new ShardKeySelector
|
||||
{
|
||||
ShardKeys = { new List<ShardKey> { currentDate! } }
|
||||
}
|
||||
);
|
||||
}
|
||||
```
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
```go
|
||||
csvUrl := "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv"
|
||||
|
||||
shardKeyDescriptions, err := client.ListShardKeys(context.Background(), collectionName)
|
||||
|
||||
// Retrieve a list of existing shard keys in the collection
|
||||
existingShardKeys := make(map[string]bool)
|
||||
for _, desc := range shardKeyDescriptions {
|
||||
existingShardKeys[desc.Key.GetKeyword()] = true
|
||||
}
|
||||
|
||||
denseModel := "sentence-transformers/all-MiniLM-L6-v2"
|
||||
batchSize := 100
|
||||
var currentDate string
|
||||
var buffer []*qdrant.PointStruct
|
||||
|
||||
err = parseCSV(csvUrl, func(row CSVRow) {
|
||||
text := row.Text
|
||||
datetime := row.Datetime
|
||||
shardDate := datetime[:10] // Extract YYYY-MM-DD
|
||||
|
||||
if shardDate != currentDate {
|
||||
// Flush buffer for the previous date before switching
|
||||
if len(buffer) > 0 {
|
||||
client.Upsert(context.Background(), &qdrant.UpsertPoints{
|
||||
CollectionName: collectionName,
|
||||
Points: buffer,
|
||||
ShardKeySelector: &qdrant.ShardKeySelector{
|
||||
ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey(currentDate)},
|
||||
},
|
||||
})
|
||||
buffer = nil
|
||||
}
|
||||
|
||||
// Create shard for the new date if it doesn't exist yet
|
||||
if !existingShardKeys[shardDate] {
|
||||
client.CreateShardKey(context.Background(), collectionName, &qdrant.CreateShardKey{
|
||||
ShardKey: qdrant.NewShardKey(shardDate),
|
||||
})
|
||||
existingShardKeys[shardDate] = true
|
||||
}
|
||||
|
||||
currentDate = shardDate
|
||||
}
|
||||
|
||||
// Add point to buffer
|
||||
buffer = append(buffer, &qdrant.PointStruct{
|
||||
Id: qdrant.NewID(uuid.New().String()),
|
||||
Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{
|
||||
"dense_vector": qdrant.NewVectorDocument(&qdrant.Document{
|
||||
Text: text,
|
||||
Model: denseModel,
|
||||
}),
|
||||
}),
|
||||
Payload: qdrant.NewValueMap(map[string]any{
|
||||
"text": text,
|
||||
"datetime": datetime,
|
||||
}),
|
||||
})
|
||||
|
||||
// Flush batch if buffer size exceeds batch size
|
||||
if len(buffer) >= batchSize {
|
||||
client.Upsert(context.Background(), &qdrant.UpsertPoints{
|
||||
CollectionName: collectionName,
|
||||
Points: buffer,
|
||||
ShardKeySelector: &qdrant.ShardKeySelector{
|
||||
ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey(currentDate)},
|
||||
},
|
||||
})
|
||||
buffer = nil
|
||||
}
|
||||
})
|
||||
|
||||
// Flush remaining partial batch
|
||||
if len(buffer) > 0 {
|
||||
client.Upsert(context.Background(), &qdrant.UpsertPoints{
|
||||
CollectionName: collectionName,
|
||||
Points: buffer,
|
||||
ShardKeySelector: &qdrant.ShardKeySelector{
|
||||
ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey(currentDate)},
|
||||
},
|
||||
})
|
||||
}
|
||||
```
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
```java
|
||||
String csvUrl = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv";
|
||||
|
||||
// Retrieve a list of existing shard keys in the collection
|
||||
var shardKeyDescriptions = client.listShardKeysAsync(collectionName).get();
|
||||
Set<String> existingShardKeys = new HashSet<>();
|
||||
for (var desc : shardKeyDescriptions) {
|
||||
existingShardKeys.add(desc.getKey().getKeyword());
|
||||
}
|
||||
|
||||
String denseModel = "sentence-transformers/all-MiniLM-L6-v2";
|
||||
int batchSize = 100;
|
||||
String currentDate = null;
|
||||
List<PointStruct> buffer = new ArrayList<>();
|
||||
|
||||
try (var stream = parseCSV(csvUrl)) {
|
||||
for (var row : (Iterable<CsvRow>) stream::iterator) {
|
||||
String text = row.text;
|
||||
String datetime = row.datetime;
|
||||
String shardDate = datetime.substring(0, 10); // Extract YYYY-MM-DD
|
||||
|
||||
if (!shardDate.equals(currentDate)) {
|
||||
// Flush buffer for the previous date before switching
|
||||
if (!buffer.isEmpty()) {
|
||||
client.upsertAsync(
|
||||
UpsertPoints.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.addAllPoints(buffer)
|
||||
.setShardKeySelector(shardKeySelector(currentDate))
|
||||
.build()
|
||||
).get();
|
||||
buffer.clear();
|
||||
}
|
||||
|
||||
// Create shard for the new date if it doesn't exist yet
|
||||
if (!existingShardKeys.contains(shardDate)) {
|
||||
client.createShardKeyAsync(
|
||||
CreateShardKeyRequest.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.setRequest(CreateShardKey.newBuilder()
|
||||
.setShardKey(shardKey(shardDate))
|
||||
.build())
|
||||
.build()
|
||||
).get();
|
||||
existingShardKeys.add(shardDate);
|
||||
}
|
||||
|
||||
currentDate = shardDate;
|
||||
}
|
||||
|
||||
// Add point to buffer
|
||||
buffer.add(
|
||||
PointStruct.newBuilder()
|
||||
.setId(id(UUID.randomUUID()))
|
||||
.setVectors(namedVectors(Map.of(
|
||||
"dense_vector",
|
||||
vector(Document.newBuilder()
|
||||
.setText(text)
|
||||
.setModel(denseModel)
|
||||
.build()))))
|
||||
.putAllPayload(Map.of("text", value(text), "datetime", value(datetime)))
|
||||
.build());
|
||||
|
||||
// Flush batch if buffer size exceeds batch size
|
||||
if (buffer.size() >= batchSize) {
|
||||
client.upsertAsync(
|
||||
UpsertPoints.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.addAllPoints(buffer)
|
||||
.setShardKeySelector(shardKeySelector(currentDate))
|
||||
.build()
|
||||
).get();
|
||||
buffer.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Flush remaining partial batch
|
||||
if (!buffer.isEmpty()) {
|
||||
client.upsertAsync(
|
||||
UpsertPoints.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.addAllPoints(buffer)
|
||||
.setShardKeySelector(shardKeySelector(currentDate))
|
||||
.build()
|
||||
).get();
|
||||
}
|
||||
```
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
```python
|
||||
from qdrant_client.http.models import PointStruct, Document
|
||||
import uuid
|
||||
|
||||
csv_url = 'https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv'
|
||||
|
||||
# Retrieve a list of existing shard keys in the collection
|
||||
existing_shard_keys = list(client.list_shard_keys(collection_name=collection_name).shard_keys)
|
||||
|
||||
dense_model = "sentence-transformers/all-MiniLM-L6-v2"
|
||||
batch_size = 100
|
||||
current_date = None
|
||||
buffer: list[PointStruct] = []
|
||||
|
||||
for row in parse_csv(csv_url):
|
||||
shard_date = row['datetime'][:10] # Extract YYYY-MM-DD
|
||||
|
||||
if shard_date != current_date:
|
||||
# Flush buffer for the previous date before switching
|
||||
if buffer:
|
||||
client.upload_points(
|
||||
collection_name=collection_name,
|
||||
points=buffer,
|
||||
shard_key_selector=current_date,
|
||||
)
|
||||
buffer = []
|
||||
|
||||
# Create shard for the new date if it doesn't exist yet
|
||||
if shard_date not in existing_shard_keys:
|
||||
client.create_shard_key(collection_name, shard_date)
|
||||
existing_shard_keys.append(shard_date)
|
||||
|
||||
current_date = shard_date
|
||||
|
||||
# Add point to buffer
|
||||
buffer.append(PointStruct(
|
||||
id=uuid.uuid4().hex,
|
||||
payload={"text": row['text'], "datetime": row['datetime']},
|
||||
vector={"dense_vector": Document(text=row["text"], model=dense_model)}
|
||||
))
|
||||
|
||||
# Flush batch if buffer size exceeds batch size
|
||||
if len(buffer) >= batch_size:
|
||||
client.upload_points(
|
||||
collection_name=collection_name,
|
||||
points=buffer,
|
||||
shard_key_selector=current_date,
|
||||
)
|
||||
buffer = []
|
||||
|
||||
# Flush remaining partial batch
|
||||
if buffer:
|
||||
client.upload_points(
|
||||
collection_name=collection_name,
|
||||
points=buffer,
|
||||
shard_key_selector=current_date,
|
||||
)
|
||||
```
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
```rust
|
||||
let csv_url = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv";
|
||||
|
||||
// Retrieve a list of existing shard keys in the collection
|
||||
let response = client.list_shard_keys(collection_name).await?;
|
||||
let mut existing_shard_keys: HashSet<String> = response
|
||||
.shard_keys
|
||||
.into_iter()
|
||||
.filter_map(|d| {
|
||||
d.key?.key.and_then(|k| match k {
|
||||
shard_key::Key::Keyword(s) => Some(s),
|
||||
_ => None,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
let dense_model = "sentence-transformers/all-MiniLM-L6-v2";
|
||||
let batch_size = 100;
|
||||
let mut current_date = String::new();
|
||||
let mut buffer: Vec<PointStruct> = Vec::new();
|
||||
|
||||
for row in parse_csv(csv_url)? {
|
||||
let row = row?;
|
||||
let text = row.text;
|
||||
let datetime = row.datetime;
|
||||
let shard_date = datetime[..10].to_string(); // Extract YYYY-MM-DD
|
||||
|
||||
if shard_date != current_date {
|
||||
// Flush buffer for the previous date before switching
|
||||
if !buffer.is_empty() {
|
||||
client
|
||||
.upsert_points(
|
||||
UpsertPointsBuilder::new(
|
||||
collection_name,
|
||||
std::mem::take(&mut buffer),
|
||||
)
|
||||
.shard_key_selector(current_date.clone()),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
// Create shard for the new date if it doesn't exist yet
|
||||
if !existing_shard_keys.contains(&shard_date) {
|
||||
client
|
||||
.create_shard_key(
|
||||
CreateShardKeyRequestBuilder::new(collection_name).request(
|
||||
CreateShardKeyBuilder::default().shard_key(shard_date.clone()),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
existing_shard_keys.insert(shard_date.clone());
|
||||
}
|
||||
|
||||
current_date = shard_date;
|
||||
}
|
||||
|
||||
// Add point to buffer
|
||||
buffer.push(PointStruct::new(
|
||||
uuid::Uuid::new_v4().to_string(),
|
||||
HashMap::from([(
|
||||
"dense_vector".to_string(),
|
||||
DocumentBuilder::new(&text, dense_model).build(),
|
||||
)]),
|
||||
[("text", text.into()), ("datetime", datetime.into())],
|
||||
));
|
||||
|
||||
// Flush batch if buffer size exceeds batch size
|
||||
if buffer.len() >= batch_size {
|
||||
client
|
||||
.upsert_points(
|
||||
UpsertPointsBuilder::new(collection_name, std::mem::take(&mut buffer))
|
||||
.shard_key_selector(current_date.clone()),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
|
||||
// Flush remaining partial batch
|
||||
if !buffer.is_empty() {
|
||||
client
|
||||
.upsert_points(
|
||||
UpsertPointsBuilder::new(collection_name, buffer)
|
||||
.shard_key_selector(current_date.clone()),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
```
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
```typescript
|
||||
const csvUrl = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv";
|
||||
|
||||
// Retrieve a list of existing shard keys in the collection
|
||||
const shardKeysResult = await client.listShardKeys(collectionName);
|
||||
const existingShardKeys = new Set((shardKeysResult.shard_keys ?? []).map((d) => String(d.key)));
|
||||
|
||||
const denseModel = "sentence-transformers/all-MiniLM-L6-v2";
|
||||
const batchSize = 100;
|
||||
let currentDate = "";
|
||||
let buffer: Extract<Parameters<typeof client.upsert>[1], { points: unknown }>['points'] = [];
|
||||
|
||||
for await (const { text: postText, datetime } of parseCSV(csvUrl)) {
|
||||
const shardDate = datetime.slice(0, 10); // Extract YYYY-MM-DD
|
||||
|
||||
if (shardDate !== currentDate) {
|
||||
// Flush buffer for the previous date before switching
|
||||
if (buffer.length > 0) {
|
||||
await client.upsert(collectionName, { points: buffer, shard_key: currentDate });
|
||||
buffer = [];
|
||||
}
|
||||
|
||||
// Create shard for the new date if it doesn't exist yet
|
||||
if (!existingShardKeys.has(shardDate)) {
|
||||
await client.createShardKey(collectionName, { shard_key: shardDate });
|
||||
existingShardKeys.add(shardDate);
|
||||
}
|
||||
|
||||
currentDate = shardDate;
|
||||
}
|
||||
|
||||
// Add point to buffer
|
||||
buffer.push({
|
||||
id: crypto.randomUUID(),
|
||||
vector: { dense_vector: { text: postText, model: denseModel } },
|
||||
payload: { text: postText, datetime },
|
||||
});
|
||||
|
||||
// Flush batch if buffer size exceeds batch size
|
||||
if (buffer.length >= batchSize) {
|
||||
await client.upsert(collectionName, { points: buffer, shard_key: currentDate });
|
||||
buffer = [];
|
||||
}
|
||||
}
|
||||
|
||||
// Flush remaining partial batch
|
||||
if (buffer.length > 0) {
|
||||
await client.upsert(collectionName, { points: buffer, shard_key: currentDate });
|
||||
}
|
||||
```
|
||||
@@ -0,0 +1,292 @@
|
||||
package snippet
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/qdrant/go-client/qdrant"
|
||||
)
|
||||
|
||||
// @block-start parse-csv
|
||||
type CSVRow struct {
|
||||
Text string
|
||||
Datetime string
|
||||
}
|
||||
|
||||
func parseCSV(url string, fn func(CSVRow)) error {
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
csvReader := csv.NewReader(resp.Body)
|
||||
headers, err := csvReader.Read()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
textIdx, datetimeIdx := -1, -1
|
||||
for i, h := range headers {
|
||||
switch h {
|
||||
case "text":
|
||||
textIdx = i
|
||||
case "datetime":
|
||||
datetimeIdx = i
|
||||
}
|
||||
}
|
||||
|
||||
for {
|
||||
row, err := csvReader.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fn(CSVRow{Text: row[textIdx], Datetime: row[datetimeIdx]})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
// @block-end parse-csv
|
||||
|
||||
func Main() {
|
||||
// @hide-start
|
||||
QDRANT_URL := ""
|
||||
QDRANT_API_KEY := ""
|
||||
// @hide-end
|
||||
|
||||
// @block-start initialize-client
|
||||
client, err := qdrant.NewClient(&qdrant.Config{
|
||||
Host: QDRANT_URL,
|
||||
APIKey: QDRANT_API_KEY,
|
||||
UseTLS: true,
|
||||
})
|
||||
// @block-end initialize-client
|
||||
|
||||
// @hide-start
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// @hide-end
|
||||
|
||||
// @block-start create-collection
|
||||
collectionName := "my_collection"
|
||||
|
||||
exists, err := client.CollectionExists(context.Background(), collectionName)
|
||||
if err != nil { panic(err) } // @hide
|
||||
if exists {
|
||||
client.DeleteCollection(context.Background(), collectionName)
|
||||
}
|
||||
|
||||
client.CreateCollection(context.Background(), &qdrant.CreateCollection{
|
||||
CollectionName: collectionName,
|
||||
VectorsConfig: qdrant.NewVectorsConfigMap(
|
||||
map[string]*qdrant.VectorParams{
|
||||
"dense_vector": {
|
||||
Size: 384,
|
||||
Distance: qdrant.Distance_Cosine,
|
||||
},
|
||||
},
|
||||
),
|
||||
ShardingMethod: qdrant.ShardingMethod_Custom.Enum(),
|
||||
})
|
||||
// @block-end create-collection
|
||||
|
||||
// @block-start upload-vectors
|
||||
csvUrl := "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv"
|
||||
|
||||
shardKeyDescriptions, err := client.ListShardKeys(context.Background(), collectionName)
|
||||
if err != nil { panic(err) } // @hide
|
||||
|
||||
// Retrieve a list of existing shard keys in the collection
|
||||
existingShardKeys := make(map[string]bool)
|
||||
for _, desc := range shardKeyDescriptions {
|
||||
existingShardKeys[desc.Key.GetKeyword()] = true
|
||||
}
|
||||
|
||||
denseModel := "sentence-transformers/all-MiniLM-L6-v2"
|
||||
batchSize := 100
|
||||
var currentDate string
|
||||
var buffer []*qdrant.PointStruct
|
||||
|
||||
err = parseCSV(csvUrl, func(row CSVRow) {
|
||||
text := row.Text
|
||||
datetime := row.Datetime
|
||||
shardDate := datetime[:10] // Extract YYYY-MM-DD
|
||||
|
||||
if shardDate != currentDate {
|
||||
// Flush buffer for the previous date before switching
|
||||
if len(buffer) > 0 {
|
||||
client.Upsert(context.Background(), &qdrant.UpsertPoints{
|
||||
CollectionName: collectionName,
|
||||
Points: buffer,
|
||||
ShardKeySelector: &qdrant.ShardKeySelector{
|
||||
ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey(currentDate)},
|
||||
},
|
||||
})
|
||||
buffer = nil
|
||||
}
|
||||
|
||||
// Create shard for the new date if it doesn't exist yet
|
||||
if !existingShardKeys[shardDate] {
|
||||
client.CreateShardKey(context.Background(), collectionName, &qdrant.CreateShardKey{
|
||||
ShardKey: qdrant.NewShardKey(shardDate),
|
||||
})
|
||||
existingShardKeys[shardDate] = true
|
||||
}
|
||||
|
||||
currentDate = shardDate
|
||||
}
|
||||
|
||||
// Add point to buffer
|
||||
buffer = append(buffer, &qdrant.PointStruct{
|
||||
Id: qdrant.NewID(uuid.New().String()),
|
||||
Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{
|
||||
"dense_vector": qdrant.NewVectorDocument(&qdrant.Document{
|
||||
Text: text,
|
||||
Model: denseModel,
|
||||
}),
|
||||
}),
|
||||
Payload: qdrant.NewValueMap(map[string]any{
|
||||
"text": text,
|
||||
"datetime": datetime,
|
||||
}),
|
||||
})
|
||||
|
||||
// Flush batch if buffer size exceeds batch size
|
||||
if len(buffer) >= batchSize {
|
||||
client.Upsert(context.Background(), &qdrant.UpsertPoints{
|
||||
CollectionName: collectionName,
|
||||
Points: buffer,
|
||||
ShardKeySelector: &qdrant.ShardKeySelector{
|
||||
ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey(currentDate)},
|
||||
},
|
||||
})
|
||||
buffer = nil
|
||||
}
|
||||
})
|
||||
if err != nil { panic(err) } // @hide
|
||||
|
||||
// Flush remaining partial batch
|
||||
if len(buffer) > 0 {
|
||||
client.Upsert(context.Background(), &qdrant.UpsertPoints{
|
||||
CollectionName: collectionName,
|
||||
Points: buffer,
|
||||
ShardKeySelector: &qdrant.ShardKeySelector{
|
||||
ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey(currentDate)},
|
||||
},
|
||||
})
|
||||
}
|
||||
// @block-end upload-vectors
|
||||
|
||||
// @block-start search-single-shard
|
||||
queryText := "coffee"
|
||||
|
||||
result, err := client.Query(context.Background(), &qdrant.QueryPoints{
|
||||
CollectionName: collectionName,
|
||||
Query: qdrant.NewQueryDocument(&qdrant.Document{Text: queryText, Model: denseModel}),
|
||||
Using: qdrant.PtrOf("dense_vector"),
|
||||
Limit: qdrant.PtrOf(uint64(5)),
|
||||
ShardKeySelector: &qdrant.ShardKeySelector{
|
||||
ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey("2026-04-07")},
|
||||
},
|
||||
})
|
||||
|
||||
// @hide-start
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// @hide-end
|
||||
|
||||
for _, hit := range result {
|
||||
fmt.Println(hit)
|
||||
}
|
||||
// @block-end search-single-shard
|
||||
|
||||
// @block-start search-multiple-shards
|
||||
result, err = client.Query(context.Background(), &qdrant.QueryPoints{
|
||||
CollectionName: collectionName,
|
||||
Query: qdrant.NewQueryDocument(&qdrant.Document{Text: queryText, Model: denseModel}),
|
||||
Using: qdrant.PtrOf("dense_vector"),
|
||||
Limit: qdrant.PtrOf(uint64(5)),
|
||||
ShardKeySelector: &qdrant.ShardKeySelector{
|
||||
ShardKeys: []*qdrant.ShardKey{
|
||||
qdrant.NewShardKey("2026-04-06"),
|
||||
qdrant.NewShardKey("2026-04-07"),
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// @hide-start
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// @hide-end
|
||||
|
||||
for _, hit := range result {
|
||||
fmt.Println(hit)
|
||||
}
|
||||
// @block-end search-multiple-shards
|
||||
|
||||
// @block-start search-all-shards
|
||||
result, err = client.Query(context.Background(), &qdrant.QueryPoints{
|
||||
CollectionName: collectionName,
|
||||
Query: qdrant.NewQueryDocument(&qdrant.Document{Text: queryText, Model: denseModel}),
|
||||
Using: qdrant.PtrOf("dense_vector"),
|
||||
Limit: qdrant.PtrOf(uint64(5)),
|
||||
})
|
||||
|
||||
// @hide-start
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// @hide-end
|
||||
|
||||
for _, hit := range result {
|
||||
fmt.Println(hit)
|
||||
}
|
||||
// @block-end search-all-shards
|
||||
|
||||
// @block-start pruning-shards
|
||||
today := "2026-04-08"
|
||||
t, _ := time.Parse("2006-01-02", today)
|
||||
oldestShardKey := t.AddDate(0, 0, -7).Format("2006-01-02")
|
||||
|
||||
client.CreateShardKey(context.Background(), collectionName, &qdrant.CreateShardKey{
|
||||
ShardKey: qdrant.NewShardKey(today),
|
||||
})
|
||||
client.DeleteShardKey(context.Background(), collectionName, &qdrant.DeleteShardKey{
|
||||
ShardKey: qdrant.NewShardKey(oldestShardKey),
|
||||
})
|
||||
// @block-end pruning-shards
|
||||
|
||||
// @block-start ingest-new-data
|
||||
client.Upsert(context.Background(), &qdrant.UpsertPoints{
|
||||
CollectionName: collectionName,
|
||||
Points: []*qdrant.PointStruct{
|
||||
{
|
||||
Id: qdrant.NewID(uuid.New().String()),
|
||||
Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{
|
||||
"dense_vector": qdrant.NewVectorDocument(&qdrant.Document{
|
||||
Text: "The best way to start a Wednesday is with a cup of coffee",
|
||||
Model: denseModel,
|
||||
}),
|
||||
}),
|
||||
Payload: qdrant.NewValueMap(map[string]any{
|
||||
"text": "The best way to start a Wednesday is with a cup of coffee",
|
||||
"datetime": "2026-04-08T07:57:47",
|
||||
}),
|
||||
},
|
||||
},
|
||||
ShardKeySelector: &qdrant.ShardKeySelector{
|
||||
ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey(today)},
|
||||
},
|
||||
})
|
||||
// @block-end ingest-new-data
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
package com.example.snippets_amalgamation;
|
||||
|
||||
import static io.qdrant.client.PointIdFactory.id;
|
||||
import static io.qdrant.client.QueryFactory.nearest;
|
||||
import static io.qdrant.client.ShardKeyFactory.shardKey;
|
||||
import static io.qdrant.client.ShardKeySelectorFactory.shardKeySelector;
|
||||
import static io.qdrant.client.ValueFactory.value;
|
||||
import static io.qdrant.client.VectorFactory.vector;
|
||||
import static io.qdrant.client.VectorsFactory.namedVectors;
|
||||
|
||||
import io.qdrant.client.QdrantClient;
|
||||
import io.qdrant.client.QdrantGrpcClient;
|
||||
import io.qdrant.client.grpc.Collections.CreateCollection;
|
||||
import io.qdrant.client.grpc.Collections.CreateShardKey;
|
||||
import io.qdrant.client.grpc.Collections.CreateShardKeyRequest;
|
||||
import io.qdrant.client.grpc.Collections.DeleteShardKey;
|
||||
import io.qdrant.client.grpc.Collections.DeleteShardKeyRequest;
|
||||
import io.qdrant.client.grpc.Collections.Distance;
|
||||
import io.qdrant.client.grpc.Collections.ShardingMethod;
|
||||
import io.qdrant.client.grpc.Collections.VectorParams;
|
||||
import io.qdrant.client.grpc.Collections.VectorParamsMap;
|
||||
import io.qdrant.client.grpc.Collections.VectorsConfig;
|
||||
import io.qdrant.client.grpc.Points.Document;
|
||||
import io.qdrant.client.grpc.Points.PointStruct;
|
||||
import io.qdrant.client.grpc.Points.QueryPoints;
|
||||
import io.qdrant.client.grpc.Points.ShardKeySelector;
|
||||
import io.qdrant.client.grpc.Points.UpsertPoints;
|
||||
import java.io.BufferedReader;
|
||||
import java.io.InputStreamReader;
|
||||
import java.net.URL;
|
||||
import java.time.LocalDate;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
public class Snippet {
|
||||
|
||||
// @block-start parse-csv
|
||||
static class CsvRow {
|
||||
final String text;
|
||||
final String datetime;
|
||||
CsvRow(String text, String datetime) { this.text = text; this.datetime = datetime; }
|
||||
}
|
||||
|
||||
static Stream<CsvRow> parseCSV(String url) throws Exception {
|
||||
Function<String, List<String>> parseCsvLine = line -> {
|
||||
List<String> fields = new ArrayList<>();
|
||||
boolean inQuotes = false;
|
||||
var sb = new StringBuilder();
|
||||
for (char c : line.toCharArray()) {
|
||||
if (c == '"') {
|
||||
inQuotes = !inQuotes;
|
||||
} else if (c == ',' && !inQuotes) {
|
||||
fields.add(sb.toString());
|
||||
sb.setLength(0);
|
||||
} else {
|
||||
sb.append(c);
|
||||
}
|
||||
}
|
||||
fields.add(sb.toString());
|
||||
return fields;
|
||||
};
|
||||
|
||||
var reader = new BufferedReader(new InputStreamReader(new URL(url).openStream()));
|
||||
String headerLine = reader.readLine();
|
||||
List<String> headers = List.of(headerLine.split(","));
|
||||
int textIdx = headers.indexOf("text");
|
||||
int datetimeIdx = headers.indexOf("datetime");
|
||||
|
||||
return reader.lines()
|
||||
.map(line -> {
|
||||
List<String> fields = parseCsvLine.apply(line);
|
||||
return new CsvRow(fields.get(textIdx), fields.get(datetimeIdx));
|
||||
})
|
||||
.onClose(() -> { try { reader.close(); } catch (Exception ignored) {} });
|
||||
}
|
||||
// @block-end parse-csv
|
||||
|
||||
public static void run() throws Exception {
|
||||
// @hide-start
|
||||
String QDRANT_URL = "";
|
||||
String QDRANT_API_KEY = "";
|
||||
// @hide-end
|
||||
|
||||
// @block-start initialize-client
|
||||
QdrantClient client =
|
||||
new QdrantClient(
|
||||
QdrantGrpcClient.newBuilder(QDRANT_URL, 6334, true)
|
||||
.withApiKey(QDRANT_API_KEY)
|
||||
.build());
|
||||
// @block-end initialize-client
|
||||
|
||||
// @block-start create-collection
|
||||
String collectionName = "my_collection";
|
||||
|
||||
if (client.collectionExistsAsync(collectionName).get()) {
|
||||
client.deleteCollectionAsync(collectionName).get();
|
||||
}
|
||||
|
||||
client.createCollectionAsync(
|
||||
CreateCollection.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.setVectorsConfig(VectorsConfig.newBuilder().setParamsMap(
|
||||
VectorParamsMap.newBuilder().putAllMap(Map.of(
|
||||
"dense_vector",
|
||||
VectorParams.newBuilder()
|
||||
.setSize(384)
|
||||
.setDistance(Distance.Cosine)
|
||||
.build()))))
|
||||
.setShardingMethod(ShardingMethod.Custom)
|
||||
.build()
|
||||
).get();
|
||||
// @block-end create-collection
|
||||
|
||||
// @block-start upload-vectors
|
||||
String csvUrl = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv";
|
||||
|
||||
// Retrieve a list of existing shard keys in the collection
|
||||
var shardKeyDescriptions = client.listShardKeysAsync(collectionName).get();
|
||||
Set<String> existingShardKeys = new HashSet<>();
|
||||
for (var desc : shardKeyDescriptions) {
|
||||
existingShardKeys.add(desc.getKey().getKeyword());
|
||||
}
|
||||
|
||||
String denseModel = "sentence-transformers/all-MiniLM-L6-v2";
|
||||
int batchSize = 100;
|
||||
String currentDate = null;
|
||||
List<PointStruct> buffer = new ArrayList<>();
|
||||
|
||||
try (var stream = parseCSV(csvUrl)) {
|
||||
for (var row : (Iterable<CsvRow>) stream::iterator) {
|
||||
String text = row.text;
|
||||
String datetime = row.datetime;
|
||||
String shardDate = datetime.substring(0, 10); // Extract YYYY-MM-DD
|
||||
|
||||
if (!shardDate.equals(currentDate)) {
|
||||
// Flush buffer for the previous date before switching
|
||||
if (!buffer.isEmpty()) {
|
||||
client.upsertAsync(
|
||||
UpsertPoints.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.addAllPoints(buffer)
|
||||
.setShardKeySelector(shardKeySelector(currentDate))
|
||||
.build()
|
||||
).get();
|
||||
buffer.clear();
|
||||
}
|
||||
|
||||
// Create shard for the new date if it doesn't exist yet
|
||||
if (!existingShardKeys.contains(shardDate)) {
|
||||
client.createShardKeyAsync(
|
||||
CreateShardKeyRequest.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.setRequest(CreateShardKey.newBuilder()
|
||||
.setShardKey(shardKey(shardDate))
|
||||
.build())
|
||||
.build()
|
||||
).get();
|
||||
existingShardKeys.add(shardDate);
|
||||
}
|
||||
|
||||
currentDate = shardDate;
|
||||
}
|
||||
|
||||
// Add point to buffer
|
||||
buffer.add(
|
||||
PointStruct.newBuilder()
|
||||
.setId(id(UUID.randomUUID()))
|
||||
.setVectors(namedVectors(Map.of(
|
||||
"dense_vector",
|
||||
vector(Document.newBuilder()
|
||||
.setText(text)
|
||||
.setModel(denseModel)
|
||||
.build()))))
|
||||
.putAllPayload(Map.of("text", value(text), "datetime", value(datetime)))
|
||||
.build());
|
||||
|
||||
// Flush batch if buffer size exceeds batch size
|
||||
if (buffer.size() >= batchSize) {
|
||||
client.upsertAsync(
|
||||
UpsertPoints.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.addAllPoints(buffer)
|
||||
.setShardKeySelector(shardKeySelector(currentDate))
|
||||
.build()
|
||||
).get();
|
||||
buffer.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Flush remaining partial batch
|
||||
if (!buffer.isEmpty()) {
|
||||
client.upsertAsync(
|
||||
UpsertPoints.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.addAllPoints(buffer)
|
||||
.setShardKeySelector(shardKeySelector(currentDate))
|
||||
.build()
|
||||
).get();
|
||||
}
|
||||
// @block-end upload-vectors
|
||||
|
||||
// @block-start search-single-shard
|
||||
String queryText = "coffee";
|
||||
|
||||
var result = client.queryAsync(
|
||||
QueryPoints.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.setQuery(nearest(Document.newBuilder().setText(queryText).setModel(denseModel).build()))
|
||||
.setUsing("dense_vector")
|
||||
.setLimit(5)
|
||||
.setShardKeySelector(shardKeySelector("2026-04-07"))
|
||||
.build()
|
||||
).get();
|
||||
|
||||
for (var hit : result) {
|
||||
System.out.println(hit);
|
||||
}
|
||||
// @block-end search-single-shard
|
||||
|
||||
// @block-start search-multiple-shards
|
||||
result = client.queryAsync(
|
||||
QueryPoints.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.setQuery(nearest(Document.newBuilder().setText(queryText).setModel(denseModel).build()))
|
||||
.setUsing("dense_vector")
|
||||
.setLimit(5)
|
||||
.setShardKeySelector(ShardKeySelector.newBuilder()
|
||||
.addShardKeys(shardKey("2026-04-06"))
|
||||
.addShardKeys(shardKey("2026-04-07"))
|
||||
.build())
|
||||
.build()
|
||||
).get();
|
||||
|
||||
for (var hit : result) {
|
||||
System.out.println(hit);
|
||||
}
|
||||
// @block-end search-multiple-shards
|
||||
|
||||
// @block-start search-all-shards
|
||||
result = client.queryAsync(
|
||||
QueryPoints.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.setQuery(nearest(Document.newBuilder().setText(queryText).setModel(denseModel).build()))
|
||||
.setUsing("dense_vector")
|
||||
.setLimit(5)
|
||||
.build()
|
||||
).get();
|
||||
|
||||
for (var hit : result) {
|
||||
System.out.println(hit);
|
||||
}
|
||||
// @block-end search-all-shards
|
||||
|
||||
// @block-start pruning-shards
|
||||
String today = "2026-04-08";
|
||||
String oldestShardKey = LocalDate.parse(today).minusDays(7).toString();
|
||||
|
||||
client.createShardKeyAsync(
|
||||
CreateShardKeyRequest.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.setRequest(CreateShardKey.newBuilder()
|
||||
.setShardKey(shardKey(today))
|
||||
.build())
|
||||
.build()
|
||||
).get();
|
||||
|
||||
client.deleteShardKeyAsync(
|
||||
DeleteShardKeyRequest.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.setRequest(DeleteShardKey.newBuilder()
|
||||
.setShardKey(shardKey(oldestShardKey))
|
||||
.build())
|
||||
.build()
|
||||
).get();
|
||||
// @block-end pruning-shards
|
||||
|
||||
// @block-start ingest-new-data
|
||||
client.upsertAsync(
|
||||
UpsertPoints.newBuilder()
|
||||
.setCollectionName(collectionName)
|
||||
.addAllPoints(List.of(
|
||||
PointStruct.newBuilder()
|
||||
.setId(id(UUID.randomUUID()))
|
||||
.setVectors(namedVectors(Map.of(
|
||||
"dense_vector",
|
||||
vector(Document.newBuilder()
|
||||
.setText("The best way to start a Wednesday is with a cup of coffee")
|
||||
.setModel(denseModel)
|
||||
.build()))))
|
||||
.putAllPayload(Map.of(
|
||||
"text", value("The best way to start a Wednesday is with a cup of coffee"),
|
||||
"datetime", value("2026-04-08T07:57:47")))
|
||||
.build()))
|
||||
.setShardKeySelector(shardKeySelector(today))
|
||||
.build()
|
||||
).get();
|
||||
// @block-end ingest-new-data
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
# @hide-start
|
||||
# mypy: disable-error-code="arg-type"
|
||||
QDRANT_URL=""
|
||||
QDRANT_API_KEY=""
|
||||
# @hide-end
|
||||
|
||||
# @block-start initialize-client
|
||||
from qdrant_client import QdrantClient, models
|
||||
|
||||
client = QdrantClient(
|
||||
url=QDRANT_URL,
|
||||
api_key=QDRANT_API_KEY,
|
||||
cloud_inference=True
|
||||
)
|
||||
# @block-end initialize-client
|
||||
|
||||
# @block-start create-collection
|
||||
from qdrant_client import models
|
||||
|
||||
collection_name = "my_collection"
|
||||
|
||||
if client.collection_exists(collection_name=collection_name):
|
||||
client.delete_collection(collection_name=collection_name)
|
||||
|
||||
client.create_collection(
|
||||
collection_name=collection_name,
|
||||
vectors_config={
|
||||
"dense_vector": models.VectorParams(
|
||||
size=384, distance=models.Distance.COSINE
|
||||
)
|
||||
},
|
||||
sharding_method=models.ShardingMethod.CUSTOM
|
||||
)
|
||||
# @block-end create-collection
|
||||
|
||||
# @block-start parse-csv
|
||||
import csv
|
||||
import urllib.request
|
||||
|
||||
def parse_csv(url):
|
||||
with urllib.request.urlopen(url) as response:
|
||||
reader = csv.DictReader(line.decode('utf-8') for line in response)
|
||||
yield from reader
|
||||
# @block-end parse-csv
|
||||
|
||||
# @block-start upload-vectors
|
||||
from qdrant_client.http.models import PointStruct, Document
|
||||
import uuid
|
||||
|
||||
csv_url = 'https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv'
|
||||
|
||||
# Retrieve a list of existing shard keys in the collection
|
||||
existing_shard_keys = list(client.list_shard_keys(collection_name=collection_name).shard_keys)
|
||||
|
||||
dense_model = "sentence-transformers/all-MiniLM-L6-v2"
|
||||
batch_size = 100
|
||||
current_date = None
|
||||
buffer: list[PointStruct] = []
|
||||
|
||||
for row in parse_csv(csv_url):
|
||||
shard_date = row['datetime'][:10] # Extract YYYY-MM-DD
|
||||
|
||||
if shard_date != current_date:
|
||||
# Flush buffer for the previous date before switching
|
||||
if buffer:
|
||||
client.upload_points(
|
||||
collection_name=collection_name,
|
||||
points=buffer,
|
||||
shard_key_selector=current_date,
|
||||
)
|
||||
buffer = []
|
||||
|
||||
# Create shard for the new date if it doesn't exist yet
|
||||
if shard_date not in existing_shard_keys:
|
||||
client.create_shard_key(collection_name, shard_date)
|
||||
existing_shard_keys.append(shard_date)
|
||||
|
||||
current_date = shard_date
|
||||
|
||||
# Add point to buffer
|
||||
buffer.append(PointStruct(
|
||||
id=uuid.uuid4().hex,
|
||||
payload={"text": row['text'], "datetime": row['datetime']},
|
||||
vector={"dense_vector": Document(text=row["text"], model=dense_model)}
|
||||
))
|
||||
|
||||
# Flush batch if buffer size exceeds batch size
|
||||
if len(buffer) >= batch_size:
|
||||
client.upload_points(
|
||||
collection_name=collection_name,
|
||||
points=buffer,
|
||||
shard_key_selector=current_date,
|
||||
)
|
||||
buffer = []
|
||||
|
||||
# Flush remaining partial batch
|
||||
if buffer:
|
||||
client.upload_points(
|
||||
collection_name=collection_name,
|
||||
points=buffer,
|
||||
shard_key_selector=current_date,
|
||||
)
|
||||
# @block-end upload-vectors
|
||||
|
||||
# @block-start search-single-shard
|
||||
query_text = "coffee"
|
||||
|
||||
resp = client.query_points(
|
||||
collection_name=collection_name,
|
||||
query=Document(text=query_text, model=dense_model),
|
||||
using="dense_vector",
|
||||
limit=5,
|
||||
shard_key_selector="2026-04-07"
|
||||
)
|
||||
print(resp)
|
||||
# @block-end search-single-shard
|
||||
|
||||
# @block-start search-multiple-shards
|
||||
resp = client.query_points(
|
||||
collection_name=collection_name,
|
||||
query=Document(text=query_text, model=dense_model),
|
||||
using="dense_vector",
|
||||
limit=5,
|
||||
shard_key_selector=["2026-04-06","2026-04-07"]
|
||||
)
|
||||
print(resp)
|
||||
# @block-end search-multiple-shards
|
||||
|
||||
# @block-start search-all-shards
|
||||
resp = client.query_points(
|
||||
collection_name=collection_name,
|
||||
query=Document(text=query_text, model=dense_model),
|
||||
using="dense_vector",
|
||||
limit=5,
|
||||
)
|
||||
print(resp)
|
||||
# @block-end search-all-shards
|
||||
|
||||
# @block-start pruning-shards
|
||||
from datetime import date, timedelta
|
||||
|
||||
today = "2026-04-08"
|
||||
oldest_shard_key = (date.fromisoformat(today) - timedelta(days=7)).isoformat()
|
||||
|
||||
client.create_shard_key(collection_name, today)
|
||||
client.delete_shard_key(collection_name, oldest_shard_key)
|
||||
# @block-end pruning-shards
|
||||
|
||||
# @block-start ingest-new-data
|
||||
client.upsert(
|
||||
collection_name=collection_name,
|
||||
points=[PointStruct(
|
||||
id=uuid.uuid4().hex,
|
||||
payload={"text": "The best way to start a Wednesday is with a cup of coffee", "datetime": "2026-04-08T07:57:47"},
|
||||
vector={
|
||||
"dense_vector": Document(text="The best way to start a Wednesday is with a cup of coffee", model=dense_model)
|
||||
})],
|
||||
shard_key_selector=today
|
||||
)
|
||||
# @block-end ingest-new-data
|
||||
@@ -0,0 +1,260 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use chrono::NaiveDate;
|
||||
use qdrant_client::Qdrant;
|
||||
use qdrant_client::qdrant::{
|
||||
CreateCollectionBuilder, CreateShardKeyBuilder, CreateShardKeyRequestBuilder,
|
||||
DeleteShardKeyRequestBuilder, Distance, Document, DocumentBuilder,
|
||||
PointStruct, Query, QueryPointsBuilder, ShardKeySelector, ShardingMethod,
|
||||
UpsertPointsBuilder, VectorParamsBuilder, VectorsConfigBuilder, shard_key,
|
||||
};
|
||||
|
||||
pub async fn main() -> anyhow::Result<()> {
|
||||
// @hide-start
|
||||
let QDRANT_URL = "";
|
||||
let QDRANT_API_KEY = "";
|
||||
// @hide-end
|
||||
|
||||
// @block-start initialize-client
|
||||
let client = Qdrant::from_url(QDRANT_URL)
|
||||
.api_key(QDRANT_API_KEY)
|
||||
.build()?;
|
||||
// @block-end initialize-client
|
||||
|
||||
// @block-start create-collection
|
||||
let collection_name = "my_collection";
|
||||
|
||||
if client.collection_exists(collection_name).await? {
|
||||
client.delete_collection(collection_name).await?;
|
||||
}
|
||||
|
||||
let mut vectors_config = VectorsConfigBuilder::default();
|
||||
vectors_config.add_named_vector_params(
|
||||
"dense_vector",
|
||||
VectorParamsBuilder::new(384, Distance::Cosine),
|
||||
);
|
||||
|
||||
client
|
||||
.create_collection(
|
||||
CreateCollectionBuilder::new(collection_name)
|
||||
.vectors_config(vectors_config)
|
||||
.sharding_method(ShardingMethod::Custom.into()),
|
||||
)
|
||||
.await?;
|
||||
// @block-end create-collection
|
||||
|
||||
// @block-start parse-csv
|
||||
struct CsvRow {
|
||||
text: String,
|
||||
datetime: String,
|
||||
}
|
||||
|
||||
fn parse_csv(url: &str) -> anyhow::Result<impl Iterator<Item = anyhow::Result<CsvRow>>> {
|
||||
let reader = ureq::get(url).call()?.into_body().into_reader();
|
||||
let mut rdr = csv::Reader::from_reader(reader);
|
||||
let headers = rdr.headers()?.clone();
|
||||
let text_idx = headers.iter().position(|h| h == "text").unwrap();
|
||||
let datetime_idx = headers.iter().position(|h| h == "datetime").unwrap();
|
||||
let iter = rdr.into_records().map(move |result| {
|
||||
let record = result?;
|
||||
Ok(CsvRow {
|
||||
text: record[text_idx].to_string(),
|
||||
datetime: record[datetime_idx].to_string(),
|
||||
})
|
||||
});
|
||||
Ok(iter)
|
||||
}
|
||||
// @block-end parse-csv
|
||||
|
||||
// @block-start upload-vectors
|
||||
let csv_url = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv";
|
||||
|
||||
// Retrieve a list of existing shard keys in the collection
|
||||
let response = client.list_shard_keys(collection_name).await?;
|
||||
let mut existing_shard_keys: HashSet<String> = response
|
||||
.shard_keys
|
||||
.into_iter()
|
||||
.filter_map(|d| {
|
||||
d.key?.key.and_then(|k| match k {
|
||||
shard_key::Key::Keyword(s) => Some(s),
|
||||
_ => None,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
let dense_model = "sentence-transformers/all-MiniLM-L6-v2";
|
||||
let batch_size = 100;
|
||||
let mut current_date = String::new();
|
||||
let mut buffer: Vec<PointStruct> = Vec::new();
|
||||
|
||||
for row in parse_csv(csv_url)? {
|
||||
let row = row?;
|
||||
let text = row.text;
|
||||
let datetime = row.datetime;
|
||||
let shard_date = datetime[..10].to_string(); // Extract YYYY-MM-DD
|
||||
|
||||
if shard_date != current_date {
|
||||
// Flush buffer for the previous date before switching
|
||||
if !buffer.is_empty() {
|
||||
client
|
||||
.upsert_points(
|
||||
UpsertPointsBuilder::new(
|
||||
collection_name,
|
||||
std::mem::take(&mut buffer),
|
||||
)
|
||||
.shard_key_selector(current_date.clone()),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
// Create shard for the new date if it doesn't exist yet
|
||||
if !existing_shard_keys.contains(&shard_date) {
|
||||
client
|
||||
.create_shard_key(
|
||||
CreateShardKeyRequestBuilder::new(collection_name).request(
|
||||
CreateShardKeyBuilder::default().shard_key(shard_date.clone()),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
existing_shard_keys.insert(shard_date.clone());
|
||||
}
|
||||
|
||||
current_date = shard_date;
|
||||
}
|
||||
|
||||
// Add point to buffer
|
||||
buffer.push(PointStruct::new(
|
||||
uuid::Uuid::new_v4().to_string(),
|
||||
HashMap::from([(
|
||||
"dense_vector".to_string(),
|
||||
DocumentBuilder::new(&text, dense_model).build(),
|
||||
)]),
|
||||
[("text", text.into()), ("datetime", datetime.into())],
|
||||
));
|
||||
|
||||
// Flush batch if buffer size exceeds batch size
|
||||
if buffer.len() >= batch_size {
|
||||
client
|
||||
.upsert_points(
|
||||
UpsertPointsBuilder::new(collection_name, std::mem::take(&mut buffer))
|
||||
.shard_key_selector(current_date.clone()),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
|
||||
// Flush remaining partial batch
|
||||
if !buffer.is_empty() {
|
||||
client
|
||||
.upsert_points(
|
||||
UpsertPointsBuilder::new(collection_name, buffer)
|
||||
.shard_key_selector(current_date.clone()),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
// @block-end upload-vectors
|
||||
|
||||
// @block-start search-single-shard
|
||||
let query_text = "coffee";
|
||||
|
||||
let result = client
|
||||
.query(
|
||||
QueryPointsBuilder::new(collection_name)
|
||||
.query(Query::new_nearest(Document::new(query_text, dense_model)))
|
||||
.using("dense_vector")
|
||||
.limit(5)
|
||||
.shard_key_selector("2026-04-07".to_string()),
|
||||
)
|
||||
.await?;
|
||||
|
||||
for hit in result.result {
|
||||
println!("{:?}", hit);
|
||||
}
|
||||
// @block-end search-single-shard
|
||||
|
||||
// @block-start search-multiple-shards
|
||||
let result = client
|
||||
.query(
|
||||
QueryPointsBuilder::new(collection_name)
|
||||
.query(Query::new_nearest(Document::new(query_text, dense_model)))
|
||||
.using("dense_vector")
|
||||
.limit(5)
|
||||
.shard_key_selector(ShardKeySelector {
|
||||
shard_keys: vec![
|
||||
"2026-04-06".to_string().into(),
|
||||
"2026-04-07".to_string().into(),
|
||||
],
|
||||
fallback: None,
|
||||
}),
|
||||
)
|
||||
.await?;
|
||||
|
||||
for hit in result.result {
|
||||
println!("{:?}", hit);
|
||||
}
|
||||
// @block-end search-multiple-shards
|
||||
|
||||
// @block-start search-all-shards
|
||||
let result = client
|
||||
.query(
|
||||
QueryPointsBuilder::new(collection_name)
|
||||
.query(Query::new_nearest(Document::new(query_text, dense_model)))
|
||||
.using("dense_vector")
|
||||
.limit(5),
|
||||
)
|
||||
.await?;
|
||||
|
||||
for hit in result.result {
|
||||
println!("{:?}", hit);
|
||||
}
|
||||
// @block-end search-all-shards
|
||||
|
||||
// @block-start pruning-shards
|
||||
let today = "2026-04-08";
|
||||
let oldest_shard_key = (NaiveDate::parse_from_str(today, "%Y-%m-%d")?
|
||||
- chrono::Duration::days(7))
|
||||
.to_string();
|
||||
|
||||
client
|
||||
.create_shard_key(
|
||||
CreateShardKeyRequestBuilder::new(collection_name)
|
||||
.request(CreateShardKeyBuilder::default().shard_key(today.to_string())),
|
||||
)
|
||||
.await?;
|
||||
|
||||
client
|
||||
.delete_shard_key(
|
||||
DeleteShardKeyRequestBuilder::new(collection_name)
|
||||
.key(shard_key::Key::Keyword(oldest_shard_key)),
|
||||
)
|
||||
.await?;
|
||||
// @block-end pruning-shards
|
||||
|
||||
// @block-start ingest-new-data
|
||||
client
|
||||
.upsert_points(
|
||||
UpsertPointsBuilder::new(
|
||||
collection_name,
|
||||
vec![PointStruct::new(
|
||||
uuid::Uuid::new_v4().to_string(),
|
||||
HashMap::from([(
|
||||
"dense_vector".to_string(),
|
||||
DocumentBuilder::new(
|
||||
"The best way to start a Wednesday is with a cup of coffee",
|
||||
dense_model,
|
||||
)
|
||||
.build(),
|
||||
)]),
|
||||
[
|
||||
("text", "The best way to start a Wednesday is with a cup of coffee".into()),
|
||||
("datetime", "2026-04-08T07:57:47".into()),
|
||||
],
|
||||
)],
|
||||
)
|
||||
.shard_key_selector(today.to_string()),
|
||||
)
|
||||
.await?;
|
||||
// @block-end ingest-new-data
|
||||
|
||||
Ok(())
|
||||
}
|
||||
+210
@@ -0,0 +1,210 @@
|
||||
import { QdrantClient } from "@qdrant/js-client-rest";
|
||||
|
||||
// @hide-start
|
||||
const QDRANT_URL = "";
|
||||
const QDRANT_API_KEY = "";
|
||||
// @hide-end
|
||||
|
||||
// @block-start initialize-client
|
||||
const client = new QdrantClient({
|
||||
url: QDRANT_URL,
|
||||
apiKey: QDRANT_API_KEY,
|
||||
});
|
||||
// @block-end initialize-client
|
||||
|
||||
// @block-start create-collection
|
||||
const collectionName = "my_collection";
|
||||
|
||||
if (await client.collectionExists(collectionName)) {
|
||||
await client.deleteCollection(collectionName);
|
||||
}
|
||||
|
||||
await client.createCollection(collectionName, {
|
||||
vectors: {
|
||||
dense_vector: {
|
||||
size: 384,
|
||||
distance: "Cosine",
|
||||
},
|
||||
},
|
||||
sharding_method: "custom",
|
||||
});
|
||||
// @block-end create-collection
|
||||
|
||||
// @block-start parse-csv
|
||||
function parseCsvLine(line: string): string[] {
|
||||
const fields: string[] = [];
|
||||
let i = 0;
|
||||
while (i < line.length) {
|
||||
if (line[i] === '"') {
|
||||
i++;
|
||||
let field = "";
|
||||
while (i < line.length) {
|
||||
if (line[i] === '"' && line[i + 1] === '"') { field += '"'; i += 2; }
|
||||
else if (line[i] === '"') { i++; break; }
|
||||
else { field += line[i++]; }
|
||||
}
|
||||
fields.push(field);
|
||||
if (line[i] === ",") i++;
|
||||
} else {
|
||||
const start = i;
|
||||
while (i < line.length && line[i] !== ",") i++;
|
||||
fields.push(line.slice(start, i));
|
||||
if (i < line.length) i++;
|
||||
}
|
||||
}
|
||||
return fields;
|
||||
}
|
||||
|
||||
async function* parseCSV(url: string): AsyncGenerator<{ text: string; datetime: string }> {
|
||||
const response = await fetch(url);
|
||||
const reader = response.body!.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let remainder = "";
|
||||
let headers: string[] | null = null;
|
||||
let textIdx = -1;
|
||||
let datetimeIdx = -1;
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
const chunk = done ? "" : decoder.decode(value, { stream: true });
|
||||
const lines = (remainder + chunk).split("\n");
|
||||
remainder = done ? "" : lines.pop()!;
|
||||
|
||||
for (const line of lines) {
|
||||
if (!line.trim()) continue;
|
||||
if (headers === null) {
|
||||
headers = line.split(",");
|
||||
textIdx = headers.indexOf("text");
|
||||
datetimeIdx = headers.indexOf("datetime");
|
||||
continue;
|
||||
}
|
||||
const fields = parseCsvLine(line);
|
||||
yield { text: fields[textIdx], datetime: fields[datetimeIdx] };
|
||||
}
|
||||
|
||||
if (done) break;
|
||||
}
|
||||
}
|
||||
// @block-end parse-csv
|
||||
|
||||
// @block-start upload-vectors
|
||||
const csvUrl = "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv";
|
||||
|
||||
// Retrieve a list of existing shard keys in the collection
|
||||
const shardKeysResult = await client.listShardKeys(collectionName);
|
||||
const existingShardKeys = new Set((shardKeysResult.shard_keys ?? []).map((d) => String(d.key)));
|
||||
|
||||
const denseModel = "sentence-transformers/all-MiniLM-L6-v2";
|
||||
const batchSize = 100;
|
||||
let currentDate = "";
|
||||
let buffer: Extract<Parameters<typeof client.upsert>[1], { points: unknown }>['points'] = [];
|
||||
|
||||
for await (const { text: postText, datetime } of parseCSV(csvUrl)) {
|
||||
const shardDate = datetime.slice(0, 10); // Extract YYYY-MM-DD
|
||||
|
||||
if (shardDate !== currentDate) {
|
||||
// Flush buffer for the previous date before switching
|
||||
if (buffer.length > 0) {
|
||||
await client.upsert(collectionName, { points: buffer, shard_key: currentDate });
|
||||
buffer = [];
|
||||
}
|
||||
|
||||
// Create shard for the new date if it doesn't exist yet
|
||||
if (!existingShardKeys.has(shardDate)) {
|
||||
await client.createShardKey(collectionName, { shard_key: shardDate });
|
||||
existingShardKeys.add(shardDate);
|
||||
}
|
||||
|
||||
currentDate = shardDate;
|
||||
}
|
||||
|
||||
// Add point to buffer
|
||||
buffer.push({
|
||||
id: crypto.randomUUID(),
|
||||
vector: { dense_vector: { text: postText, model: denseModel } },
|
||||
payload: { text: postText, datetime },
|
||||
});
|
||||
|
||||
// Flush batch if buffer size exceeds batch size
|
||||
if (buffer.length >= batchSize) {
|
||||
await client.upsert(collectionName, { points: buffer, shard_key: currentDate });
|
||||
buffer = [];
|
||||
}
|
||||
}
|
||||
|
||||
// Flush remaining partial batch
|
||||
if (buffer.length > 0) {
|
||||
await client.upsert(collectionName, { points: buffer, shard_key: currentDate });
|
||||
}
|
||||
// @block-end upload-vectors
|
||||
|
||||
// @block-start search-single-shard
|
||||
const queryText = "coffee";
|
||||
|
||||
const singleShardResult = await client.query(collectionName, {
|
||||
query: { text: queryText, model: denseModel },
|
||||
using: "dense_vector",
|
||||
limit: 5,
|
||||
shard_key: "2026-04-07",
|
||||
});
|
||||
|
||||
for (const hit of singleShardResult.points) {
|
||||
console.log(hit);
|
||||
}
|
||||
// @block-end search-single-shard
|
||||
|
||||
// @block-start search-multiple-shards
|
||||
const multiShardResult = await client.query(collectionName, {
|
||||
query: { text: queryText, model: denseModel },
|
||||
using: "dense_vector",
|
||||
limit: 5,
|
||||
shard_key: ["2026-04-06", "2026-04-07"],
|
||||
});
|
||||
|
||||
for (const hit of multiShardResult.points) {
|
||||
console.log(hit);
|
||||
}
|
||||
// @block-end search-multiple-shards
|
||||
|
||||
// @block-start search-all-shards
|
||||
const allShardsResult = await client.query(collectionName, {
|
||||
query: { text: queryText, model: denseModel },
|
||||
using: "dense_vector",
|
||||
limit: 5,
|
||||
});
|
||||
|
||||
for (const hit of allShardsResult.points) {
|
||||
console.log(hit);
|
||||
}
|
||||
// @block-end search-all-shards
|
||||
|
||||
// @block-start pruning-shards
|
||||
const today = "2026-04-08";
|
||||
const oldestDate = new Date(today);
|
||||
oldestDate.setDate(oldestDate.getDate() - 7);
|
||||
const oldestShardKey = oldestDate.toISOString().slice(0, 10);
|
||||
|
||||
await client.createShardKey(collectionName, { shard_key: today });
|
||||
await client.deleteShardKey(collectionName, { shard_key: oldestShardKey });
|
||||
// @block-end pruning-shards
|
||||
|
||||
// @block-start ingest-new-data
|
||||
await client.upsert(collectionName, {
|
||||
points: [
|
||||
{
|
||||
id: crypto.randomUUID(),
|
||||
vector: {
|
||||
dense_vector: {
|
||||
text: "The best way to start a Wednesday is with a cup of coffee",
|
||||
model: denseModel,
|
||||
},
|
||||
},
|
||||
payload: {
|
||||
text: "The best way to start a Wednesday is with a cup of coffee",
|
||||
datetime: "2026-04-08T07:57:47",
|
||||
},
|
||||
},
|
||||
],
|
||||
shard_key: today,
|
||||
});
|
||||
// @block-end ingest-new-data
|
||||
Reference in New Issue
Block a user