tiered multitenancy docs

This commit is contained in:
generall
2025-11-16 21:24:34 +01:00
parent 0a1203c831
commit 31c7db602b
59 changed files with 911 additions and 334 deletions
@@ -361,113 +361,7 @@ A clear use-case for this feature is managing a multi-tenant collection, where e
To enable user-defined sharding, set `sharding_method` to `custom` during collection creation:
```http
PUT /collections/{collection_name}
{
"shard_number": 1,
"sharding_method": "custom"
// ... other collection parameters
}
```
```python
from qdrant_client import QdrantClient, models
client = QdrantClient(url="http://localhost:6333")
client.create_collection(
collection_name="{collection_name}",
shard_number=1,
sharding_method=models.ShardingMethod.CUSTOM,
# ... other collection parameters
)
```
```typescript
import { QdrantClient } from "@qdrant/js-client-rest";
const client = new QdrantClient({ host: "localhost", port: 6333 });
client.createCollection("{collection_name}", {
shard_number: 1,
sharding_method: "custom",
// ... other collection parameters
});
```
```rust
use qdrant_client::qdrant::{
CreateCollectionBuilder, Distance, ShardingMethod, VectorParamsBuilder,
};
use qdrant_client::Qdrant;
let client = Qdrant::from_url("http://localhost:6334").build()?;
client
.create_collection(
CreateCollectionBuilder::new("{collection_name}")
.vectors_config(VectorParamsBuilder::new(300, Distance::Cosine))
.shard_number(1)
.sharding_method(ShardingMethod::Custom.into()),
)
.await?;
```
```java
import static io.qdrant.client.ShardKeyFactory.shardKey;
import io.qdrant.client.QdrantClient;
import io.qdrant.client.QdrantGrpcClient;
import io.qdrant.client.grpc.Collections.CreateCollection;
import io.qdrant.client.grpc.Collections.ShardingMethod;
QdrantClient client =
new QdrantClient(QdrantGrpcClient.newBuilder("localhost", 6334, false).build());
client
.createCollectionAsync(
CreateCollection.newBuilder()
.setCollectionName("{collection_name}")
// ... other collection parameters
.setShardNumber(1)
.setShardingMethod(ShardingMethod.Custom)
.build())
.get();
```
```csharp
using Qdrant.Client;
using Qdrant.Client.Grpc;
var client = new QdrantClient("localhost", 6334);
await client.CreateCollectionAsync(
collectionName: "{collection_name}",
// ... other collection parameters
shardNumber: 1,
shardingMethod: ShardingMethod.Custom
);
```
```go
import (
"context"
"github.com/qdrant/go-client/qdrant"
)
client, err := qdrant.NewClient(&qdrant.Config{
Host: "localhost",
Port: 6334,
})
client.CreateCollection(context.Background(), &qdrant.CreateCollection{
CollectionName: "{collection_name}",
// ... other collection parameters
ShardNumber: qdrant.PtrOf(uint32(1)),
ShardingMethod: qdrant.ShardingMethod_Custom.Enum(),
})
```
{{< code-snippet path="/documentation/headless/snippets/create-collection/with-custom-sharding/" >}}
In this mode, the `shard_number` means the number of shards per shard key, where points will be distributed evenly. For example, if you have 10 shard keys and a collection config with these settings:
@@ -487,235 +381,11 @@ For large cardinality keys, it is recommended to use [partition by payload](/doc
Now you need to create custom shards ([API reference](https://api.qdrant.tech/api-reference/distributed/create-shard-key#request)):
```http
PUT /collections/{collection_name}/shards
{
"shard_key": "{shard_key}"
}
```
```python
from qdrant_client import QdrantClient, models
client = QdrantClient(url="http://localhost:6333")
client.create_shard_key("{collection_name}", "{shard_key}")
```
```typescript
import { QdrantClient } from "@qdrant/js-client-rest";
const client = new QdrantClient({ host: "localhost", port: 6333 });
client.createShardKey("{collection_name}", {
shard_key: "{shard_key}"
});
```
```rust
use qdrant_client::qdrant::{
CreateShardKeyBuilder, CreateShardKeyRequestBuilder
};
use qdrant_client::Qdrant;
let client = Qdrant::from_url("http://localhost:6334").build()?;
client
.create_shard_key(
CreateShardKeyRequestBuilder::new("{collection_name}")
.request(CreateShardKeyBuilder::default().shard_key("{shard_key".to_string())),
)
.await?;
```
```java
import static io.qdrant.client.ShardKeyFactory.shardKey;
import io.qdrant.client.QdrantClient;
import io.qdrant.client.QdrantGrpcClient;
import io.qdrant.client.grpc.Collections.CreateShardKey;
import io.qdrant.client.grpc.Collections.CreateShardKeyRequest;
QdrantClient client =
new QdrantClient(QdrantGrpcClient.newBuilder("localhost", 6334, false).build());
client.createShardKeyAsync(CreateShardKeyRequest.newBuilder()
.setCollectionName("{collection_name}")
.setRequest(CreateShardKey.newBuilder()
.setShardKey(shardKey("{shard_key}"))
.build())
.build()).get();
```
```csharp
using Qdrant.Client;
using Qdrant.Client.Grpc;
var client = new QdrantClient("localhost", 6334);
await client.CreateShardKeyAsync(
"{collection_name}",
new CreateShardKey { ShardKey = new ShardKey { Keyword = "{shard_key}", } }
);
```
```go
import (
"context"
"github.com/qdrant/go-client/qdrant"
)
client, err := qdrant.NewClient(&qdrant.Config{
Host: "localhost",
Port: 6334,
})
client.CreateShardKey(context.Background(), "{collection_name}", &qdrant.CreateShardKey{
ShardKey: qdrant.NewShardKey("{shard_key}"),
})
```
{{< code-snippet path="/documentation/headless/snippets/create-shard/create-named-shard/" >}}
To specify the shard for each point, you need to provide the `shard_key` field in the upsert request:
```http
PUT /collections/{collection_name}/points
{
"points": [
{
"id": 1111,
"vector": [0.1, 0.2, 0.3]
},
]
"shard_key": "user_1"
}
```
```python
from qdrant_client import QdrantClient, models
client = QdrantClient(url="http://localhost:6333")
client.upsert(
collection_name="{collection_name}",
points=[
models.PointStruct(
id=1111,
vector=[0.1, 0.2, 0.3],
),
],
shard_key_selector="user_1",
)
```
```typescript
import { QdrantClient } from "@qdrant/js-client-rest";
const client = new QdrantClient({ host: "localhost", port: 6333 });
client.upsert("{collection_name}", {
points: [
{
id: 1111,
vector: [0.1, 0.2, 0.3],
},
],
shard_key: "user_1",
});
```
```rust
use qdrant_client::qdrant::{PointStruct, UpsertPointsBuilder};
use qdrant_client::Payload;
client
.upsert_points(
UpsertPointsBuilder::new(
"{collection_name}",
vec![PointStruct::new(
111,
vec![0.1, 0.2, 0.3],
Payload::default(),
)],
)
.shard_key_selector("user_1".to_string()),
)
.await?;
```
```java
import java.util.List;
import static io.qdrant.client.PointIdFactory.id;
import static io.qdrant.client.ShardKeySelectorFactory.shardKeySelector;
import static io.qdrant.client.VectorsFactory.vectors;
import io.qdrant.client.QdrantClient;
import io.qdrant.client.QdrantGrpcClient;
import io.qdrant.client.grpc.Points.PointStruct;
import io.qdrant.client.grpc.Points.UpsertPoints;
QdrantClient client =
new QdrantClient(QdrantGrpcClient.newBuilder("localhost", 6334, false).build());
client
.upsertAsync(
UpsertPoints.newBuilder()
.setCollectionName("{collection_name}")
.addAllPoints(
List.of(
PointStruct.newBuilder()
.setId(id(111))
.setVectors(vectors(0.1f, 0.2f, 0.3f))
.build()))
.setShardKeySelector(shardKeySelector("user_1"))
.build())
.get();
```
```csharp
using Qdrant.Client;
using Qdrant.Client.Grpc;
var client = new QdrantClient("localhost", 6334);
await client.UpsertAsync(
collectionName: "{collection_name}",
points: new List<PointStruct>
{
new() { Id = 111, Vectors = new[] { 0.1f, 0.2f, 0.3f } }
},
shardKeySelector: new ShardKeySelector { ShardKeys = { new List<ShardKey> { "user_1" } } }
);
```
```go
import (
"context"
"github.com/qdrant/go-client/qdrant"
)
client, err := qdrant.NewClient(&qdrant.Config{
Host: "localhost",
Port: 6334,
})
client.Upsert(context.Background(), &qdrant.UpsertPoints{
CollectionName: "{collection_name}",
Points: []*qdrant.PointStruct{
{
Id: qdrant.NewIDNum(111),
Vectors: qdrant.NewVectors(0.1, 0.2, 0.3),
},
},
ShardKeySelector: &qdrant.ShardKeySelector{
ShardKeys: []*qdrant.ShardKey{
qdrant.NewShardKey("user_1"),
},
},
})
```
{{< code-snippet path="/documentation/headless/snippets/insert-points/with-custom-shard/" >}}
<aside role="alert">
Using the same point ID across multiple shard keys is <strong>not supported<sup>*</sup></strong> and should be avoided.
@@ -57,10 +57,92 @@ To implement this approach, you should:
When specified, storage structure will be organized in a way to co-locate vectors of the same tenant together, which can significantly improve performance by utilizing sequential reads during queries.
{{< figure src="/docs/defragmentation.png" alt="Tenants defragmentation with is_tenant" caption="Grouping tenants together by tenant ID, if `is_tenant=true` is used" width="80%" >}}
{{< figure src="/docs/defragmentation.png" alt="Tenants defragmentation with is_tenant" caption="Grouping tenants together by tenant ID, if `is_tenant=true` is used" width="90%" >}}
### Limitations
One downside to this approach is that global requests (without the `group_id` filter) will be slower since they will necessitate scanning all groups to identify the nearest neighbors.
## Tiered multitenancy
In some real-world applications, tenants might not be equally distributed. For example, a SaaS application might have a few large customers and many small ones.
Large tenancts might require extended resources and isolation, while small tenants should not create too much overhead.
One solution to this problem might be to introduce application-level logic to separate tenants into different collections based on their size or resource requirements.
There is, however, a downside to this approach: we might not know in advance which tenants will be large and which stay small.
In addition, application-level logic increases complexity of the system and requires additional source of truth for tenant placement management.
To address this problem, in v1.16.0 Qdrant provides a built-in mechanism for tiered multitenancy.
With tiered multitenancy, you can implement two levels of tenant isolation within a single collection, keeping small tenants together inside a shared Shard, while isolating large tenants into their own dedicated Shards.
There are 3 components in Qdrant, that allows you to implement tiered multitenancy:
- [**User-defined Sharding**](/documentation/guides/distributed_deployment/#user-defined-sharding) allows you to create named Shards within a collection. It allows to isolate large tenants into their own Shards.
- **Fallback shards** - a special routing mechanism that allows to route request to either a dedicated Shard (if it exists) or to a shared Fallback Shard. It allows to keep requests unified, without the need to know whether a tenant is dedicated or shared.
- **Tenant promotion** - a mechanism that allows to move tenants from the shared Fallback Shard to their own dedicated Shard when they grow large enough. This process is based on Qdrant's internal shard transfer mechanism, which makes promotion completely transparent for the application. Both read and write requests are supported during the promotion process.
{{< figure src="/docs/tenant-promotion.png" alt="Teired multitenancy with tenant promotion" caption="Teired multitenancy with tenant promotion" width="90%" >}}
### Configure tiered multitenancy
To take advantage of tiered multitenancy, you need to create a collection with user-defined (aka `custom`) sharding and create a Fallback Shard in it.
{{< code-snippet path="/documentation/headless/snippets/create-collection/with-custom-sharding/" >}}
Start with creating a fallback Shard, which will be used to store small tenants.
Let's name it `default`.
{{< code-snippet path="/documentation/headless/snippets/create-shard/create-named-shard-default/" >}}
Since the collection will allow both dedicated and shared tenants, we need still need to configure payload-based tenancy for this collection the same way as described in the [Partition by payload](#partition-by-payload) section above. Namely, we need to create a payload index for the `group_id` field with `is_tenant=true`.
{{< code-snippet path="/documentation/headless/snippets/create-payload-index/with-group-id-as-tenant/" >}}
### Query tiered multitenant collection
Now we can start uploading data into the collection. One important difference from the simple payload-based multitenancy is that now we need to specify the **Shard Key Selector** in each request to route requests to the correct Shard.
Shard Key Selector will specify two keys:
- `target` shard - name of the tenant's dedicated Shard (which may or may not exist).
- `fallback` shard - name of the shared Fallback Shard (in our case, `default`).
{{< code-snippet path="/documentation/headless/snippets/insert-points/with-tenant-group-id-and-fallback-shard-key/" >}}
The routing logic will work as follows:
- If the `target` Shard exists and active, the request will be routed to it.
- If the `target` Shard does not exist, the request will be routed to the `fallback` Shard.
Similarly, when querying points, we need to specify the Shard Key Selector and filter by `group_id`:
<!-- ToDo snippet -->
### Promote tenant to dedicated Shard
When a tenant grows large enough, you can promote it to its own dedicated Shard.
In order to do that, you first need to create a new Shard for the tenant:
{{< code-snippet path="/documentation/headless/snippets/create-shard/create-named-shard-for-promotion/" >}}
Note, that we create a Shard in `Partial` state, since it would still need to transfer data into it.
To initiate data transfer, there is another API method called `replicate_points`:
{{< code-snippet path="/documentation/headless/snippets/shard-transfer/with-filter/" >}}
Once transfer is completed, target Shard will become `Active`, and all requests for the tenant will be routed to it automatically.
At this point it is safe to delete the tenant's data from the shared Fallback Shard to free up space.
### Limitations
- Currently, `fallback` Shard may only contain a single shard ID on it's own. That means all small tenants must fit a single peer of the cluser. This restriction will be improved in future releases.
- Similar to collections, dedicated Shards introduce some resource overhead. It is not recommended to create more than a thousand dedicated Shards per cluster. Recommended threshold of promoting a tenant is the same as indeing threshold for a single collection, which is around 20K points.
@@ -0,0 +1,2 @@
This code snippet is used to create a collection with a User Defined Sharding (aka custom sharding) configured.
Unlike default auto-sharding, that uses hash-based sharding strategy, user-defined sharding allows to create named Shards and route requests to specific Shards based on application-level logic.
@@ -0,0 +1,13 @@
```csharp
using Qdrant.Client;
using Qdrant.Client.Grpc;
var client = new QdrantClient("localhost", 6334);
await client.CreateCollectionAsync(
collectionName: "{collection_name}",
// ... other collection parameters
shardNumber: 1,
shardingMethod: ShardingMethod.Custom
);
```
@@ -0,0 +1,19 @@
```go
import (
"context"
"github.com/qdrant/go-client/qdrant"
)
client, err := qdrant.NewClient(&qdrant.Config{
Host: "localhost",
Port: 6334,
})
client.CreateCollection(context.Background(), &qdrant.CreateCollection{
CollectionName: "{collection_name}",
// ... other collection parameters
ShardNumber: qdrant.PtrOf(uint32(1)),
ShardingMethod: qdrant.ShardingMethod_Custom.Enum(),
})
```
@@ -0,0 +1,8 @@
```http
PUT /collections/{collection_name}
{
"shard_number": 1,
"sharding_method": "custom"
// ... other collection parameters
}
```
@@ -0,0 +1,21 @@
```java
import static io.qdrant.client.ShardKeyFactory.shardKey;
import io.qdrant.client.QdrantClient;
import io.qdrant.client.QdrantGrpcClient;
import io.qdrant.client.grpc.Collections.CreateCollection;
import io.qdrant.client.grpc.Collections.ShardingMethod;
QdrantClient client =
new QdrantClient(QdrantGrpcClient.newBuilder("localhost", 6334, false).build());
client
.createCollectionAsync(
CreateCollection.newBuilder()
.setCollectionName("{collection_name}")
// ... other collection parameters
.setShardNumber(1)
.setShardingMethod(ShardingMethod.Custom)
.build())
.get();
```
@@ -0,0 +1,12 @@
```python
from qdrant_client import QdrantClient, models
client = QdrantClient(url="http://localhost:6333")
client.create_collection(
collection_name="{collection_name}",
shard_number=1,
sharding_method=models.ShardingMethod.CUSTOM,
# ... other collection parameters
)
```
@@ -0,0 +1,17 @@
```rust
use qdrant_client::qdrant::{
CreateCollectionBuilder, Distance, ShardingMethod, VectorParamsBuilder,
};
use qdrant_client::Qdrant;
let client = Qdrant::from_url("http://localhost:6334").build()?;
client
.create_collection(
CreateCollectionBuilder::new("{collection_name}")
.vectors_config(VectorParamsBuilder::new(300, Distance::Cosine))
.shard_number(1)
.sharding_method(ShardingMethod::Custom.into()),
)
.await?;
```
@@ -0,0 +1,11 @@
```typescript
import { QdrantClient } from "@qdrant/js-client-rest";
const client = new QdrantClient({ host: "localhost", port: 6333 });
client.createCollection("{collection_name}", {
shard_number: 1,
sharding_method: "custom",
// ... other collection parameters
});
```
@@ -0,0 +1,5 @@
This code snippet creates a named shard with the name "default" in a Qdrant collection.
Collection is required to be configured with `custom` sharding method to support named shards.
Once created, named shard will receive all requests that specify its name in the shard key selector.
If no named shard is specified in the request, request will be broadcasted to all shards in the collection.
@@ -0,0 +1,11 @@
```csharp
using Qdrant.Client;
using Qdrant.Client.Grpc;
var client = new QdrantClient("localhost", 6334);
await client.CreateShardKeyAsync(
"{collection_name}",
new CreateShardKey { ShardKey = new ShardKey { Keyword = "default", } }
);
```
@@ -0,0 +1,16 @@
```go
import (
"context"
"github.com/qdrant/go-client/qdrant"
)
client, err := qdrant.NewClient(&qdrant.Config{
Host: "localhost",
Port: 6334,
})
client.CreateShardKey(context.Background(), "{collection_name}", &qdrant.CreateShardKey{
ShardKey: qdrant.NewShardKey("default"),
})
```
@@ -0,0 +1,6 @@
```http
PUT /collections/{collection_name}/shards
{
"shard_key": "default"
}
```
@@ -0,0 +1,18 @@
```java
import static io.qdrant.client.ShardKeyFactory.shardKey;
import io.qdrant.client.QdrantClient;
import io.qdrant.client.QdrantGrpcClient;
import io.qdrant.client.grpc.Collections.CreateShardKey;
import io.qdrant.client.grpc.Collections.CreateShardKeyRequest;
QdrantClient client =
new QdrantClient(QdrantGrpcClient.newBuilder("localhost", 6334, false).build());
client.createShardKeyAsync(CreateShardKeyRequest.newBuilder()
.setCollectionName("{collection_name}")
.setRequest(CreateShardKey.newBuilder()
.setShardKey(shardKey("default"))
.build())
.build()).get();
```
@@ -0,0 +1,7 @@
```python
from qdrant_client import QdrantClient, models
client = QdrantClient(url="http://localhost:6333")
client.create_shard_key("{collection_name}", "default")
```
@@ -0,0 +1,15 @@
```rust
use qdrant_client::qdrant::{
CreateShardKeyBuilder, CreateShardKeyRequestBuilder
};
use qdrant_client::Qdrant;
let client = Qdrant::from_url("http://localhost:6334").build()?;
client
.create_shard_key(
CreateShardKeyRequestBuilder::new("{collection_name}")
.request(CreateShardKeyBuilder::default().shard_key("default".to_string())),
)
.await?;
```
@@ -0,0 +1,9 @@
```typescript
import { QdrantClient } from "@qdrant/js-client-rest";
const client = new QdrantClient({ host: "localhost", port: 6333 });
client.createShardKey("{collection_name}", {
shard_key: "default"
});
```
@@ -0,0 +1,5 @@
This code snippet creates a named shard with the name "user_1" in a Qdrant collection.
This shard is intended to be used as a dedicated shard for a specific tenant or user, allowing for better data isolation and management. Creation of the shard specifies initial state as `Partial`, as it needs to be populated with data before it can serve requests.
Collection is required to be configured with `custom` sharding method to support named shards.
Once created, named shard will receive all requests that specify its name in the shard key selector.
@@ -0,0 +1,14 @@
```csharp
using Qdrant.Client;
using Qdrant.Client.Grpc;
var client = new QdrantClient("localhost", 6334);
await client.CreateShardKeyAsync(
"{collection_name}",
new CreateShardKey {
ShardKey = new ShardKey { Keyword = "default" },
InitialState = ReplicaState.Partial
}
);
```
@@ -0,0 +1,21 @@
```go
import (
"context"
"github.com/qdrant/go-client/qdrant"
)
client, err := qdrant.NewClient(&qdrant.Config{
Host: "localhost",
Port: 6334,
})
client.CreateShardKey(
context.Background(),
"{collection_name}",
&qdrant.CreateShardKey{
ShardKey: qdrant.NewShardKey("default"),
InitialState: qdrant.ReplicaState_PARTIAL,
}
)
```
@@ -0,0 +1,7 @@
```http
PUT /collections/{collection_name}/shards
{
"shard_key": "user_1",
"initial_state": "Partial"
}
```
@@ -0,0 +1,20 @@
```java
import static io.qdrant.client.ShardKeyFactory.shardKey;
import io.qdrant.client.QdrantClient;
import io.qdrant.client.QdrantGrpcClient;
import io.qdrant.client.grpc.Collections.CreateShardKey;
import io.qdrant.client.grpc.Collections.CreateShardKeyRequest;
import io.qdrant.client.grpc.Collections.ReplicaState;
QdrantClient client =
new QdrantClient(QdrantGrpcClient.newBuilder("localhost", 6334, false).build());
client.createShardKeyAsync(CreateShardKeyRequest.newBuilder()
.setCollectionName("{collection_name}")
.setRequest(CreateShardKey.newBuilder()
.setShardKey(shardKey("default"))
.setInitialState(ReplicaState.PARTIAL)
.build())
.build()).get();
```
@@ -0,0 +1,11 @@
```python
from qdrant_client import QdrantClient, models
client = QdrantClient(url="http://localhost:6333")
client.create_shard_key(
"{collection_name}",
shard_key="user_1",
initial_state=models.ReplicaState.PARTIAL
)
```
@@ -0,0 +1,20 @@
```rust
use qdrant_client::qdrant::{
CreateShardKeyBuilder, CreateShardKeyRequestBuilder
};
use qdrant_client::qdrant::ReplicaState;
use qdrant_client::Qdrant;
let client = Qdrant::from_url("http://localhost:6334").build()?;
client
.create_shard_key(
CreateShardKeyRequestBuilder::new("{collection_name}")
.request(
CreateShardKeyBuilder::default()
.shard_key("user_1".to_string())
.initial_state(ReplicaState::Partial)
),
)
.await?;
```
@@ -0,0 +1,10 @@
```typescript
import { QdrantClient } from "@qdrant/js-client-rest";
const client = new QdrantClient({ host: "localhost", port: 6333 });
client.createShardKey("{collection_name}", {
shard_key: "default",
initial_state: "Partial"
});
```
@@ -0,0 +1,5 @@
This code snippet creates a named shard in a Qdrant collection.
Collection is required to be configured with `custom` sharding method to support named shards.
Once created, named shard will receive all requests that specify its name in the shard key selector.
If no named shard is specified in the request, request will be broadcasted to all shards in the collection.
@@ -0,0 +1,11 @@
```csharp
using Qdrant.Client;
using Qdrant.Client.Grpc;
var client = new QdrantClient("localhost", 6334);
await client.CreateShardKeyAsync(
"{collection_name}",
new CreateShardKey { ShardKey = new ShardKey { Keyword = "{shard_key}", } }
);
```
@@ -0,0 +1,16 @@
```go
import (
"context"
"github.com/qdrant/go-client/qdrant"
)
client, err := qdrant.NewClient(&qdrant.Config{
Host: "localhost",
Port: 6334,
})
client.CreateShardKey(context.Background(), "{collection_name}", &qdrant.CreateShardKey{
ShardKey: qdrant.NewShardKey("{shard_key}"),
})
```
@@ -0,0 +1,6 @@
```http
PUT /collections/{collection_name}/shards
{
"shard_key": "{shard_key}"
}
```
@@ -0,0 +1,18 @@
```java
import static io.qdrant.client.ShardKeyFactory.shardKey;
import io.qdrant.client.QdrantClient;
import io.qdrant.client.QdrantGrpcClient;
import io.qdrant.client.grpc.Collections.CreateShardKey;
import io.qdrant.client.grpc.Collections.CreateShardKeyRequest;
QdrantClient client =
new QdrantClient(QdrantGrpcClient.newBuilder("localhost", 6334, false).build());
client.createShardKeyAsync(CreateShardKeyRequest.newBuilder()
.setCollectionName("{collection_name}")
.setRequest(CreateShardKey.newBuilder()
.setShardKey(shardKey("{shard_key}"))
.build())
.build()).get();
```
@@ -0,0 +1,7 @@
```python
from qdrant_client import QdrantClient, models
client = QdrantClient(url="http://localhost:6333")
client.create_shard_key("{collection_name}", "{shard_key}")
```
@@ -0,0 +1,15 @@
```rust
use qdrant_client::qdrant::{
CreateShardKeyBuilder, CreateShardKeyRequestBuilder
};
use qdrant_client::Qdrant;
let client = Qdrant::from_url("http://localhost:6334").build()?;
client
.create_shard_key(
CreateShardKeyRequestBuilder::new("{collection_name}")
.request(CreateShardKeyBuilder::default().shard_key("{shard_key}".to_string())),
)
.await?;
```
@@ -0,0 +1,9 @@
```typescript
import { QdrantClient } from "@qdrant/js-client-rest";
const client = new QdrantClient({ host: "localhost", port: 6333 });
client.createShardKey("{collection_name}", {
shard_key: "{shard_key}"
});
```
@@ -0,0 +1 @@
This code snippet demonstrates how to insert point into Qdrant collection with custom sharding enabled. Request have an additional parameter, `shard_key`, which specifies routing information for the point being inserted.
@@ -0,0 +1,15 @@
```csharp
using Qdrant.Client;
using Qdrant.Client.Grpc;
var client = new QdrantClient("localhost", 6334);
await client.UpsertAsync(
collectionName: "{collection_name}",
points: new List<PointStruct>
{
new() { Id = 111, Vectors = new[] { 0.1f, 0.2f, 0.3f } }
},
shardKeySelector: new ShardKeySelector { ShardKeys = { new List<ShardKey> { "user_1" } } }
);
```
@@ -0,0 +1,27 @@
```go
import (
"context"
"github.com/qdrant/go-client/qdrant"
)
client, err := qdrant.NewClient(&qdrant.Config{
Host: "localhost",
Port: 6334,
})
client.Upsert(context.Background(), &qdrant.UpsertPoints{
CollectionName: "{collection_name}",
Points: []*qdrant.PointStruct{
{
Id: qdrant.NewIDNum(111),
Vectors: qdrant.NewVectors(0.1, 0.2, 0.3),
},
},
ShardKeySelector: &qdrant.ShardKeySelector{
ShardKeys: []*qdrant.ShardKey{
qdrant.NewShardKey("user_1"),
},
},
})
```
@@ -0,0 +1,12 @@
```http
PUT /collections/{collection_name}/points
{
"points": [
{
"id": 1111,
"vector": [0.1, 0.2, 0.3]
},
]
"shard_key": "user_1"
}
```
@@ -0,0 +1,30 @@
```java
import java.util.List;
import static io.qdrant.client.PointIdFactory.id;
import static io.qdrant.client.ShardKeySelectorFactory.shardKeySelector;
import static io.qdrant.client.VectorsFactory.vectors;
import io.qdrant.client.QdrantClient;
import io.qdrant.client.QdrantGrpcClient;
import io.qdrant.client.grpc.Points.PointStruct;
import io.qdrant.client.grpc.Points.UpsertPoints;
QdrantClient client =
new QdrantClient(QdrantGrpcClient.newBuilder("localhost", 6334, false).build());
client
.upsertAsync(
UpsertPoints.newBuilder()
.setCollectionName("{collection_name}")
.addAllPoints(
List.of(
PointStruct.newBuilder()
.setId(id(111))
.setVectors(vectors(0.1f, 0.2f, 0.3f))
.build()))
.setShardKeySelector(shardKeySelector("user_1"))
.build()
)
.get();
```
@@ -0,0 +1,16 @@
```python
from qdrant_client import QdrantClient, models
client = QdrantClient(url="http://localhost:6333")
client.upsert(
collection_name="{collection_name}",
points=[
models.PointStruct(
id=1111,
vector=[0.1, 0.2, 0.3],
),
],
shard_key_selector="user_1",
)
```
@@ -0,0 +1,18 @@
```rust
use qdrant_client::qdrant::{PointStruct, UpsertPointsBuilder};
use qdrant_client::Payload;
client
.upsert_points(
UpsertPointsBuilder::new(
"{collection_name}",
vec![PointStruct::new(
111,
vec![0.1, 0.2, 0.3],
Payload::default(),
)],
)
.shard_key_selector("user_1".to_string()),
)
.await?;
```
@@ -0,0 +1,15 @@
```typescript
import { QdrantClient } from "@qdrant/js-client-rest";
const client = new QdrantClient({ host: "localhost", port: 6333 });
client.upsert("{collection_name}", {
points: [
{
id: 1111,
vector: [0.1, 0.2, 0.3],
},
],
shard_key: "user_1",
});
```
@@ -0,0 +1,2 @@
This code snippet is for a PUT request to insert points into a collection, where each point has an ID, a payload containing a group ID, and a vector. The code illustrates partitioning vectors by user to ensure that each user can only access their own vectors. It emphasizes adding a `group_id` field to each vector in the collection, facilitating user-specific data access control. Additionally, it suggests using an appropriate naming convention for the key in the payload for flexibility in data structures.
In addition, the snippet includes a shard key selector, allowing a dymamic routing between shared and dedicated shards based on the existance of `target` shard in the collection.
@@ -0,0 +1,23 @@
```csharp
using Qdrant.Client;
using Qdrant.Client.Grpc;
var client = new QdrantClient("localhost", 6334);
await client.UpsertAsync(
collectionName: "{collection_name}",
points: new List<PointStruct>
{
new()
{
Id = 1,
Vectors = new[] { 0.9f, 0.1f, 0.1f },
Payload = { ["group_id"] = "user_1" }
}
},
shardKeySelector: new ShardKeySelector {
ShardKeys = { new List<ShardKey> { "user_1" } },
Fallback = new ShardKey { Keyword = "default" }
}
);
```
@@ -0,0 +1,29 @@
```go
import (
"context"
"github.com/qdrant/go-client/qdrant"
)
client, err := qdrant.NewClient(&qdrant.Config{
Host: "localhost",
Port: 6334,
})
client.Upsert(context.Background(), &qdrant.UpsertPoints{
CollectionName: "{collection_name}",
Points: []*qdrant.PointStruct{
{
Id: qdrant.NewIDNum(1),
Vectors: qdrant.NewVectors(0.9, 0.1, 0.1),
Payload: qdrant.NewValueMap(map[string]any{"group_id": "user_1"}),
}
},
ShardKeySelector: &qdrant.ShardKeySelector{
ShardKeys: []*qdrant.ShardKey{
qdrant.NewShardKey("user_1"),
},
Fallback: qdrant.NewShardKey("default"),
},
})
```
@@ -0,0 +1,16 @@
```http
PUT /collections/{collection_name}/points
{
"points": [
{
"id": 1,
"payload": {"group_id": "user_1"},
"vector": [0.9, 0.1, 0.1]
}
],
"shard_key": {
"fallback": "default",
"target": "user_1"
}
}
```
@@ -0,0 +1,37 @@
```java
import java.util.List;
import java.util.Map;
import io.qdrant.client.QdrantClient;
import io.qdrant.client.QdrantGrpcClient;
import io.qdrant.client.grpc.Points.PointStruct;
import io.qdrant.client.ShardKeyFactory.shardKey;
import io.qdrant.client.grpc.Points.ShardKeySelector;
QdrantClient client =
new QdrantClient(QdrantGrpcClient.newBuilder("localhost", 6334, false).build());
ShardKeySelector.Builder shardkeySelectorBuilder = ShardKeySelector.newBuilder();
shardkeySelectorBuilder.addShardKeys(shardKey("user_1"));
shardkeySelectorBuilder.setFallback(shardKey("default"));
ShardKeySelector shardKeySelector = shardkeySelectorBuilder.build();
client
.upsertAsync(
UpsertPoints.newBuilder()
.setCollectionName("{collection_name}")
.addAllPoints(
List.of(
PointStruct.newBuilder()
.setId(id(1))
.setVectors(vectors(0.9f, 0.1f, 0.1f))
.putAllPayload(Map.of("group_id", value("user_1")))
.build()
)
)
.setShardKeySelector(shardKeySelector)
.build()
)
.get();
```
@@ -0,0 +1,16 @@
```python
client.upsert(
collection_name="{collection_name}",
points=[
models.PointStruct(
id=1,
payload={"group_id": "user_1"},
vector=[0.9, 0.1, 0.1],
),
],
shard_key_selector=models.ShardKeyWithFallback(
target="user_1",
fallback="default"
)
)
```
@@ -0,0 +1,26 @@
```rust
use qdrant_client::qdrant::{PointStruct, UpsertPointsBuilder};
use qdrant_client::Qdrant;
let client = Qdrant::from_url("http://localhost:6334").build()?;
let shard_key_selector = ShardKeySelectorBuilder::with_shard_key("user_1")
.fallback("default")
.build();
client
.upsert_points(
UpsertPointsBuilder::new(
"{collection_name}",
vec![
PointStruct::new(
1,
vec![0.9, 0.1, 0.1],
[("group_id", "user_1".into())]
),
],
)
.shard_key_selector(shard_key_selector),
)
.await?;
```
@@ -0,0 +1,19 @@
```typescript
import { QdrantClient } from "@qdrant/js-client-rest";
const client = new QdrantClient({ host: "localhost", port: 6333 });
client.upsert("{collection_name}", {
points: [
{
id: 1,
payload: { group_id: "user_1" },
vector: [0.9, 0.1, 0.1],
}
],
shard_key: {
target: "user_1",
fallback: "default"
}
});
```
@@ -0,0 +1,3 @@
This request is used to proote tenant from shared Shard to dedicated shard.
It takes two shard keys - one indicated where to transfer from, another where to thatsfer to,
as well as filtering condition which defines a subset of points to transfer.
@@ -0,0 +1,11 @@
```csharp
using Qdrant.Client;
using Qdrant.Client.Grpc;
using static Qdrant.Client.Grpc.Conditions;
var client = new QdrantClient("localhost", 6334);
TODO!
```
@@ -0,0 +1,14 @@
```go
import (
"context"
"github.com/qdrant/go-client/qdrant"
)
client, err := qdrant.NewClient(&qdrant.Config{
Host: "localhost",
Port: 6334,
})
TODO!
```
@@ -0,0 +1,17 @@
```http
POST /collections/{collection_name}/cluster
{
"replicate_points": {
"filter": {
"must": {
"key": "group_id",
"match": {
"value": "user_1"
}
}
},
"from_shard_key": "default",
"to_shard_key": "user_1"
}
}
```
@@ -0,0 +1,20 @@
```java
import java.util.List;
import static io.qdrant.client.ConditionFactory.matchKeyword;
import static io.qdrant.client.QueryFactory.nearest;
import io.qdrant.client.QdrantClient;
import io.qdrant.client.QdrantGrpcClient;
import io.qdrant.client.grpc.Points.Filter;
QdrantClient client =
new QdrantClient(QdrantGrpcClient.newBuilder("localhost", 6334, false).build());
// .setFilter(Filter.newBuilder().addMust(matchKeyword("group_id", "user_1")).build())
TODO!
```
@@ -0,0 +1,26 @@
```python
from qdrant_client import QdrantClient, models
client = QdrantClient(url="http://localhost:6333")
client.cluster_collection_update(
collection_name="{collection_name}",
cluster_operation=models.ReplicatePointsOperation(
replicate_points=models.ReplicatePoints(
from_shard_key="default",
to_shard_key="user_1",
filter=models.Filter(
must=[
models.FieldCondition(
key="group_id",
match=models.MatchValue(
value="user_1",
)
)
]
)
)
)
)
```
@@ -0,0 +1,27 @@
```rust
use qdrant_client::qdrant::{
update_collection_cluster_setup_request::Operation, Condition, Filter,
ReplicatePointsBuilder, ShardKey, UpdateCollectionClusterSetupRequest,
};
use qdrant_client::Qdrant;
let client = Qdrant::from_url("http://localhost:6334").build()?;
client
.update_collection_cluster_setup(UpdateCollectionClusterSetupRequest {
collection_name: "{collection_name}".to_string(),
operation: Some(Operation::ReplicatePoints(
ReplicatePointsBuilder::new(
ShardKey::from("default"),
ShardKey::from("user_1"),
)
.filter(Filter::must([Condition::matches(
"group_id",
"user_1".to_string(),
)]))
.build(),
)),
timeout: None,
})
.await?;
```
@@ -0,0 +1,20 @@
```typescript
import { QdrantClient } from "@qdrant/js-client-rest";
const client = new QdrantClient({ host: "localhost", port: 6333 });
client.updateCollectionCluster("{collection_name}", {
replicate_points: {
filter: {
must: {
key: "group_id",
match: {
value: "user_1"
}
}
},
from_shard_key: "default",
to_shard_key: "user_1"
}
});
```
Binary file not shown.

After

Width:  |  Height:  |  Size: 32 KiB