mirror of
https://github.com/qdrant/landing_page.git
synced 2026-09-26 14:38:30 +02:00
Update Bulk Upload tutorial for best practices (#2407)
* Make Bulk Upload tutorial code snippets testable * Align Bulk Upload tutorial with agent skill * Update Bulk Upload tutorial * Update title * Remove 'Disable Indexing' section
This commit is contained in:
Abdon Pijpelink
@@ -253,9 +253,7 @@ Read about our recommendations in the [bulk upload](/documentation/tutorials-dev
|
||||
|
||||
There is no universal recommended batch size. The optimum depends on your vector dimensionality, payload size, cluster configuration, and available memory. You should benchmark different batch sizes against your own setup to find what works best.
|
||||
|
||||
A good starting point is 16 to 32 MB per request. This translates to approximately 100 points per batch when dealing with large payloads, or up to 1000 points per batch for pure vectors. However, if operations within a batch are inherently expensive, such as updates impacting many points or updates by filter, it is more efficient to send individual requests.
|
||||
|
||||
A useful pattern for large-scale bulk loads is **staged indexing**: disable HNSW graph construction during upload [by setting the HNSQ `m` parameter to `0`](/documentation/tutorials-develop/bulk-upload/#defer-hnsw-graph-construction-m-0), upload in batches using the upsert API, then restore the threshold to trigger background indexing once the load is complete. This avoids optimizer thrashing and significantly improves throughput during the initial load.
|
||||
A good starting point is 64 to 256 points per batch. However, if operations within a batch are inherently expensive, such as updates impacting many points or updates by filter, it is more efficient to send individual requests.
|
||||
|
||||
See also: [Bulk Operations](/documentation/tutorials-develop/bulk-upload/)
|
||||
|
||||
|
||||
@@ -2,6 +2,6 @@
|
||||
| :--- | :--- | :--- | :--- | :--- |
|
||||
| [Build a Semantic Search API](/documentation/tutorials-develop/neural-search/) | Deploy a search service for company descriptions. | <span class="pill">FastAPI</span> | 30m | <span class="text-green">Beginner</span> |
|
||||
| [Build a Hybrid Search API](/documentation/tutorials-develop/hybrid-search-fastembed/) | Combine dense and sparse search. | <span class="pill">FastAPI</span> | 20m | <span class="text-green">Beginner</span> |
|
||||
| [Bulk Operations](/documentation/tutorials-develop/bulk-upload/) | High-scale ingestion approaches. | <span class="pill">Python</span> | 20m | <span class="text-yellow">Intermediate</span> |
|
||||
| [Bulk Upload](/documentation/tutorials-develop/bulk-upload/) | High-scale ingestion approaches. | <span class="pill">Any</span> | 20m | <span class="text-yellow">Intermediate</span> |
|
||||
| [Async API](/documentation/tutorials-develop/async-api/) | Use Asynchronous programming for efficiency. | <span class="pill">Python</span> | 25m | <span class="text-yellow">Intermediate</span> |
|
||||
| [Semantic Search for Code](/documentation/tutorials-develop/code-search/) | Navigate codebases using vector similarity. | <span class="pill">Python</span> | 45m | <span class="text-yellow">Intermediate</span> |
|
||||
+1
@@ -0,0 +1 @@
|
||||
This code snippet is used to create a collection with two shards. Increasing the shard number allows you to distribute the collection data across multiple nodes in a cluster.
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
using Qdrant.Client;
|
||||
using Qdrant.Client.Grpc;
|
||||
|
||||
public class Snippet
|
||||
{
|
||||
public static async Task Run()
|
||||
{
|
||||
// @hide-start
|
||||
var client = new QdrantClient("localhost", 6334);
|
||||
// @hide-end
|
||||
|
||||
await client.CreateCollectionAsync(
|
||||
collectionName: "{collection_name}",
|
||||
vectorsConfig: new VectorParams { Size = 768, Distance = Distance.Cosine },
|
||||
shardNumber: 2
|
||||
);
|
||||
}
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
```csharp
|
||||
using Qdrant.Client;
|
||||
using Qdrant.Client.Grpc;
|
||||
|
||||
await client.CreateCollectionAsync(
|
||||
collectionName: "{collection_name}",
|
||||
vectorsConfig: new VectorParams { Size = 768, Distance = Distance.Cosine },
|
||||
shardNumber: 2
|
||||
);
|
||||
```
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
```go
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/qdrant/go-client/qdrant"
|
||||
)
|
||||
|
||||
client.CreateCollection(context.Background(), &qdrant.CreateCollection{
|
||||
CollectionName: "{collection_name}",
|
||||
VectorsConfig: qdrant.NewVectorsConfig(&qdrant.VectorParams{
|
||||
Size: 768,
|
||||
Distance: qdrant.Distance_Cosine,
|
||||
}),
|
||||
ShardNumber: qdrant.PtrOf(uint32(2)),
|
||||
})
|
||||
```
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
```java
|
||||
import io.qdrant.client.QdrantClient;
|
||||
import io.qdrant.client.QdrantGrpcClient;
|
||||
import io.qdrant.client.grpc.Collections.CreateCollection;
|
||||
import io.qdrant.client.grpc.Collections.Distance;
|
||||
import io.qdrant.client.grpc.Collections.VectorParams;
|
||||
import io.qdrant.client.grpc.Collections.VectorsConfig;
|
||||
|
||||
client
|
||||
.createCollectionAsync(
|
||||
CreateCollection.newBuilder()
|
||||
.setCollectionName("{collection_name}")
|
||||
.setVectorsConfig(
|
||||
VectorsConfig.newBuilder()
|
||||
.setParams(
|
||||
VectorParams.newBuilder()
|
||||
.setSize(768)
|
||||
.setDistance(Distance.Cosine)
|
||||
.build())
|
||||
.build())
|
||||
.setShardNumber(2)
|
||||
.build())
|
||||
.get();
|
||||
```
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
```python
|
||||
from qdrant_client import QdrantClient, models
|
||||
|
||||
client.create_collection(
|
||||
collection_name="{collection_name}",
|
||||
vectors_config=models.VectorParams(size=768, distance=models.Distance.COSINE),
|
||||
shard_number=2,
|
||||
)
|
||||
```
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
```rust
|
||||
use qdrant_client::qdrant::{CreateCollectionBuilder, Distance, VectorParamsBuilder};
|
||||
use qdrant_client::Qdrant;
|
||||
|
||||
client
|
||||
.create_collection(
|
||||
CreateCollectionBuilder::new("{collection_name}")
|
||||
.vectors_config(VectorParamsBuilder::new(768, Distance::Cosine))
|
||||
.shard_number(2),
|
||||
)
|
||||
.await?;
|
||||
```
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
```typescript
|
||||
import { QdrantClient } from "@qdrant/js-client-rest";
|
||||
|
||||
client.createCollection("{collection_name}", {
|
||||
vectors: {
|
||||
size: 768,
|
||||
distance: "Cosine",
|
||||
},
|
||||
shard_number: 2,
|
||||
});
|
||||
```
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
package snippet
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/qdrant/go-client/qdrant"
|
||||
)
|
||||
|
||||
func Main() {
|
||||
// @hide-start
|
||||
client, err := qdrant.NewClient(&qdrant.Config{
|
||||
Host: "localhost",
|
||||
Port: 6334,
|
||||
})
|
||||
|
||||
if err != nil { panic(err) }
|
||||
// @hide-end
|
||||
|
||||
client.CreateCollection(context.Background(), &qdrant.CreateCollection{
|
||||
CollectionName: "{collection_name}",
|
||||
VectorsConfig: qdrant.NewVectorsConfig(&qdrant.VectorParams{
|
||||
Size: 768,
|
||||
Distance: qdrant.Distance_Cosine,
|
||||
}),
|
||||
ShardNumber: qdrant.PtrOf(uint32(2)),
|
||||
})
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
```http
|
||||
PUT /collections/{collection_name}
|
||||
{
|
||||
"vectors": {
|
||||
"size": 768,
|
||||
"distance": "Cosine"
|
||||
},
|
||||
"shard_number": 2
|
||||
}
|
||||
```
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
package com.example.snippets_amalgamation;
|
||||
|
||||
import io.qdrant.client.QdrantClient;
|
||||
import io.qdrant.client.QdrantGrpcClient;
|
||||
import io.qdrant.client.grpc.Collections.CreateCollection;
|
||||
import io.qdrant.client.grpc.Collections.Distance;
|
||||
import io.qdrant.client.grpc.Collections.VectorParams;
|
||||
import io.qdrant.client.grpc.Collections.VectorsConfig;
|
||||
|
||||
public class Snippet {
|
||||
public static void run() throws Exception {
|
||||
// @hide-start
|
||||
QdrantClient client =
|
||||
new QdrantClient(QdrantGrpcClient.newBuilder("localhost", 6334, false).build());
|
||||
// @hide-end
|
||||
|
||||
client
|
||||
.createCollectionAsync(
|
||||
CreateCollection.newBuilder()
|
||||
.setCollectionName("{collection_name}")
|
||||
.setVectorsConfig(
|
||||
VectorsConfig.newBuilder()
|
||||
.setParams(
|
||||
VectorParams.newBuilder()
|
||||
.setSize(768)
|
||||
.setDistance(Distance.Cosine)
|
||||
.build())
|
||||
.build())
|
||||
.setShardNumber(2)
|
||||
.build())
|
||||
.get();
|
||||
}
|
||||
}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
from qdrant_client import QdrantClient, models
|
||||
|
||||
# @hide-start
|
||||
client = QdrantClient(url="http://localhost:6333")
|
||||
# @hide-end
|
||||
|
||||
client.create_collection(
|
||||
collection_name="{collection_name}",
|
||||
vectors_config=models.VectorParams(size=768, distance=models.Distance.COSINE),
|
||||
shard_number=2,
|
||||
)
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
use qdrant_client::qdrant::{CreateCollectionBuilder, Distance, VectorParamsBuilder};
|
||||
use qdrant_client::Qdrant;
|
||||
|
||||
pub async fn main() -> anyhow::Result<()> {
|
||||
// @hide-start
|
||||
let client = Qdrant::from_url("http://localhost:6334").build()?;
|
||||
// @hide-end
|
||||
|
||||
client
|
||||
.create_collection(
|
||||
CreateCollectionBuilder::new("{collection_name}")
|
||||
.vectors_config(VectorParamsBuilder::new(768, Distance::Cosine))
|
||||
.shard_number(2),
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
import { QdrantClient } from "@qdrant/js-client-rest";
|
||||
|
||||
// @hide-start
|
||||
const client = new QdrantClient({ host: "localhost", port: 6333 });
|
||||
// @hide-end
|
||||
|
||||
client.createCollection("{collection_name}", {
|
||||
vectors: {
|
||||
size: 768,
|
||||
distance: "Cosine",
|
||||
},
|
||||
shard_number: 2,
|
||||
});
|
||||
+1
-1
@@ -5,7 +5,7 @@ public class Snippet
|
||||
{
|
||||
public static async Task Run()
|
||||
{
|
||||
var client = new QdrantClient("localhost", 6334);
|
||||
var client = new QdrantClient("localhost", 6334); // @hide
|
||||
|
||||
await client.UpdateCollectionAsync(
|
||||
collectionName: "{collection_name}",
|
||||
|
||||
-2
@@ -2,8 +2,6 @@
|
||||
using Qdrant.Client;
|
||||
using Qdrant.Client.Grpc;
|
||||
|
||||
var client = new QdrantClient("localhost", 6334);
|
||||
|
||||
await client.UpdateCollectionAsync(
|
||||
collectionName: "{collection_name}",
|
||||
optimizersConfig: new OptimizersConfigDiff { IndexingThreshold = 10000 }
|
||||
|
||||
-5
@@ -5,11 +5,6 @@ import (
|
||||
"github.com/qdrant/go-client/qdrant"
|
||||
)
|
||||
|
||||
client, err := qdrant.NewClient(&qdrant.Config{
|
||||
Host: "localhost",
|
||||
Port: 6334,
|
||||
})
|
||||
|
||||
client.UpdateCollection(context.Background(), &qdrant.UpdateCollection{
|
||||
CollectionName: "{collection_name}",
|
||||
OptimizersConfig: &qdrant.OptimizersConfigDiff{
|
||||
|
||||
+3
-1
@@ -7,12 +7,14 @@ import (
|
||||
)
|
||||
|
||||
func Main() {
|
||||
// @hide-start
|
||||
client, err := qdrant.NewClient(&qdrant.Config{
|
||||
Host: "localhost",
|
||||
Port: 6334,
|
||||
})
|
||||
|
||||
if err != nil { panic(err) } // @hide
|
||||
if err != nil { panic(err) }
|
||||
// @hide-end
|
||||
|
||||
client.UpdateCollection(context.Background(), &qdrant.UpdateCollection{
|
||||
CollectionName: "{collection_name}",
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
---
|
||||
title: Bulk Operations
|
||||
title: Bulk Upload
|
||||
short_description: "Bulk-upload vectors into Qdrant collections efficiently by tuning indexing strategy and using high-performance client libraries."
|
||||
description: "Tutorial: bulk-upload large vector datasets into Qdrant by deferring HNSW index construction and parallelizing client uploads for maximum throughput."
|
||||
aliases:
|
||||
@@ -13,641 +13,60 @@ weight: 1
|
||||
| Time: 20 min | Level: Intermediate |
|
||||
| --- | ----------- |
|
||||
|
||||
Uploading a large-scale dataset fast might be a challenge, but Qdrant has a few tricks to help you with that.
|
||||
Uploading a large dataset quickly can be a challenge, but Qdrant provides several strategies to help.
|
||||
|
||||
The first important detail about data uploading is that the bottleneck is usually located on the client side, not on the server side.
|
||||
This means that if you are uploading a large dataset, you should prefer a high-performance client library.
|
||||
The bottleneck during data upload is usually on the client side, not the server.
|
||||
This means that if you are uploading a large dataset, you should prefer a high-performance client library. We recommend using our [Rust client library](https://github.com/qdrant/rust-client) for this purpose, as it is the fastest client library available for Qdrant.
|
||||
|
||||
We recommend using our [Rust client library](https://github.com/qdrant/rust-client) for this purpose, as it is the fastest client library available for Qdrant.
|
||||
## Batch Your Uploads
|
||||
|
||||
If you are not using Rust, you might want to consider parallelizing your upload process.
|
||||
[Upsert points in batches](/documentation/manage-data/points/#upload-points) rather than one at a time. Each request to Qdrant carries overhead: network round-trip, Write-Ahead Log (WAL) write, and internal routing. When you upload points individually, that overhead impacts the throughput.
|
||||
|
||||
## Choose an Indexing Strategy
|
||||
Aim for **64–256 points per batch**. Smaller batches under-utilize the network; larger batches can increase memory pressure on the server and raise the cost of retrying on failure. The optimal batch size depends on your data and cluster, so you may want to experiment with different sizes for best performance.
|
||||
|
||||
Qdrant incrementally builds an HNSW index for dense vectors as new data arrives. This ensures fast search, but indexing is memory- and CPU-intensive. During bulk ingestion, frequent index updates can reduce throughput and increase resource usage.
|
||||
## Parallelize Across Multiple Threads
|
||||
|
||||
To control this behavior and optimize for your system’s limits, adjust the following parameters:
|
||||
A single upload thread rarely saturates the server. Split your dataset across two to four concurrent threads, each sending its own stream of batches. This keeps Qdrant's internal write workers busy across shards and reduces total upload time.
|
||||
|
||||
| Your Goal | What to Do | Configuration |
|
||||
|-------------------------------------------|-------------------------------------------------|----------------------------------------------------|
|
||||
| Fastest upload, tolerate high RAM usage | Disable indexing completely | `indexing_threshold: 0` |
|
||||
| Low memory usage during upload | Defer HNSW graph construction (recommended) | `m: 0` |
|
||||
| Faster index availability after upload | Keep indexing enabled (default behavior) | `m: 16`, `indexing_threshold: 10000` *(default)* |
|
||||
If [your collection has multiple shards](#create-collections-with-multiple-shards), target one upload thread per shard as a starting point. Each shard has an independent WAL and update worker, so parallel streams map directly onto available write capacity.
|
||||
|
||||
Indexing must be re-enabled after upload to activate fast HNSW search if it was disabled during ingestion.
|
||||
|
||||
> The Python client's `upload_points` method [handles batching and parallelization for you](/documentation/manage-data/points/#python-client-optimizations). Pass an iterator of points and set `batch_size` and `parallel` to control throughput without managing batches manually. For other client libraries, you need to implement batching and parallelization yourself.
|
||||
|
||||
### Defer HNSW graph construction (`m: 0`)
|
||||
## Create Collections with Multiple Shards
|
||||
|
||||
For dense vectors, setting the HNSW `m` parameter to `0` disables index building entirely. Vectors will still be stored, but not indexed until you enable indexing later.
|
||||
In Qdrant, each collection is split into shards. By default, a collection has one shard, but you can specify more when creating the collection.
|
||||
By creating multiple shards, you can parallelize the upload of a large dataset. From two to four shards per machine is a reasonable number.
|
||||
|
||||
```http
|
||||
PUT /collections/{collection_name}
|
||||
{
|
||||
"vectors": {
|
||||
"size": 768,
|
||||
"distance": "Cosine"
|
||||
},
|
||||
"hnsw_config": {
|
||||
"m": 0
|
||||
}
|
||||
}
|
||||
```
|
||||
{{< code-snippet path="/documentation/headless/snippets/create-collection/with-two-shards/" >}}
|
||||
|
||||
```python
|
||||
from qdrant_client import QdrantClient, models
|
||||
## Create Payload Indexes Before Ingesting Data
|
||||
|
||||
client = QdrantClient(url="http://localhost:6333")
|
||||
If your collection uses payload indexes, [create them](/documentation/manage-data/indexing/#create-a-payload-index) before you start uploading points. Qdrant builds extra HNSW links for each payload index to optimize filtered vector search quality. If you add a payload index after the HNSW graph is already built, those links won't exist, and filtered search will fall back to slower query-time strategies until you [rebuild the graph](/documentation/manage-data/indexing/#rebuild-the-hnsw-index), which is resource-intensive and can take a long time.
|
||||
|
||||
client.create_collection(
|
||||
collection_name="{collection_name}",
|
||||
vectors_config=models.VectorParams(size=768, distance=models.Distance.COSINE),
|
||||
hnsw_config=models.HnswConfigDiff(
|
||||
m=0,
|
||||
),
|
||||
)
|
||||
```
|
||||
The correct order is:
|
||||
|
||||
```typescript
|
||||
import { QdrantClient } from "@qdrant/js-client-rest";
|
||||
1. Create the collection.
|
||||
2. Create all payload indexes.
|
||||
3. Upload your points.
|
||||
|
||||
const client = new QdrantClient({ host: "localhost", port: 6333 });
|
||||
Following this sequence means Qdrant builds the graph in a single pass, rather than having to rebuild it after the fact.
|
||||
|
||||
client.createCollection("{collection_name}", {
|
||||
vectors: {
|
||||
size: 768,
|
||||
distance: "Cosine",
|
||||
},
|
||||
hnsw_config: {
|
||||
m: 0,
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
```rust
|
||||
use qdrant_client::qdrant::{
|
||||
CreateCollectionBuilder, Distance, HnswConfigDiffBuilder, VectorParamsBuilder,
|
||||
};
|
||||
use qdrant_client::Qdrant;
|
||||
|
||||
let client = Qdrant::from_url("http://localhost:6334").build()?;
|
||||
|
||||
client
|
||||
.create_collection(
|
||||
CreateCollectionBuilder::new("{collection_name}")
|
||||
.vectors_config(VectorParamsBuilder::new(768, Distance::Cosine))
|
||||
.hnsw_config(HnswConfigDiffBuilder::default().m(0)),
|
||||
)
|
||||
.await?;
|
||||
```
|
||||
|
||||
```java
|
||||
import io.qdrant.client.QdrantClient;
|
||||
import io.qdrant.client.QdrantGrpcClient;
|
||||
import io.qdrant.client.grpc.Collections.CreateCollection;
|
||||
import io.qdrant.client.grpc.Collections.Distance;
|
||||
import io.qdrant.client.grpc.Collections.HnswConfigDiff;
|
||||
import io.qdrant.client.grpc.Collections.VectorParams;
|
||||
import io.qdrant.client.grpc.Collections.VectorsConfig;
|
||||
|
||||
QdrantClient client =
|
||||
new QdrantClient(QdrantGrpcClient.newBuilder("localhost", 6334, false).build());
|
||||
|
||||
client
|
||||
.createCollectionAsync(
|
||||
CreateCollection.newBuilder()
|
||||
.setCollectionName("{collection_name}")
|
||||
.setVectorsConfig(
|
||||
VectorsConfig.newBuilder()
|
||||
.setParams(
|
||||
VectorParams.newBuilder()
|
||||
.setSize(768)
|
||||
.setDistance(Distance.Cosine)
|
||||
.build())
|
||||
.build())
|
||||
.setHnswConfig(HnswConfigDiff.newBuilder().setM(0).build())
|
||||
.build())
|
||||
.get();
|
||||
```
|
||||
|
||||
```csharp
|
||||
using Qdrant.Client;
|
||||
using Qdrant.Client.Grpc;
|
||||
|
||||
var client = new QdrantClient("localhost", 6334);
|
||||
|
||||
await client.CreateCollectionAsync(
|
||||
collectionName: "{collection_name}",
|
||||
vectorsConfig: new VectorParams { Size = 768, Distance = Distance.Cosine },
|
||||
hnswConfig: new HnswConfigDiff { M = 0 }
|
||||
);
|
||||
```
|
||||
|
||||
```go
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/qdrant/go-client/qdrant"
|
||||
)
|
||||
|
||||
client, err := qdrant.NewClient(&qdrant.Config{
|
||||
Host: "localhost",
|
||||
Port: 6334,
|
||||
})
|
||||
|
||||
client.CreateCollection(context.Background(), &qdrant.CreateCollection{
|
||||
CollectionName: "{collection_name}",
|
||||
VectorsConfig: qdrant.NewVectorsConfig(&qdrant.VectorParams{
|
||||
Size: 768,
|
||||
Distance: qdrant.Distance_Cosine,
|
||||
}),
|
||||
HnswConfig: &qdrant.HnswConfigDiff{
|
||||
M: qdrant.PtrOf(uint64(0)),
|
||||
},
|
||||
})
|
||||
```
|
||||
|
||||
Once ingestion is complete, re-enable HNSW by setting `m` to your production value (usually 16 or 32).
|
||||
|
||||
```http
|
||||
PATCH /collections/{collection_name}
|
||||
{
|
||||
"vectors": {
|
||||
"size": 768,
|
||||
"distance": "Cosine"
|
||||
},
|
||||
"hnsw_config": {
|
||||
"m": 16
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
```python
|
||||
from qdrant_client import QdrantClient, models
|
||||
|
||||
client = QdrantClient(url="http://localhost:6333")
|
||||
|
||||
client.update_collection(
|
||||
collection_name="{collection_name}",
|
||||
vectors_config=models.VectorParams(size=768, distance=models.Distance.COSINE),
|
||||
hnsw_config=models.HnswConfigDiff(
|
||||
m=16,
|
||||
),
|
||||
)
|
||||
```
|
||||
|
||||
```typescript
|
||||
import { QdrantClient } from "@qdrant/js-client-rest";
|
||||
|
||||
const client = new QdrantClient({ host: "localhost", port: 6333 });
|
||||
|
||||
client.updateCollection("{collection_name}", {
|
||||
vectors: {
|
||||
size: 768,
|
||||
distance: "Cosine",
|
||||
},
|
||||
hnsw_config: {
|
||||
m: 16,
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
```rust
|
||||
use qdrant_client::qdrant::{
|
||||
UpdateCollectionBuilder, HnswConfigDiffBuilder,
|
||||
};
|
||||
use qdrant_client::Qdrant;
|
||||
|
||||
let client = Qdrant::from_url("http://localhost:6334").build()?;
|
||||
|
||||
client
|
||||
.update_collection(
|
||||
UpdateCollectionBuilder::new("{collection_name}")
|
||||
.hnsw_config(HnswConfigDiffBuilder::default().m(16)),
|
||||
)
|
||||
.await?;
|
||||
```
|
||||
|
||||
```java
|
||||
import io.qdrant.client.grpc.Collections.UpdateCollection;
|
||||
import io.qdrant.client.grpc.Collections.HnswConfigDiff;
|
||||
|
||||
QdrantClient client =
|
||||
new QdrantClient(QdrantGrpcClient.newBuilder("localhost", 6334, false).build());
|
||||
|
||||
client.updateCollectionAsync(
|
||||
UpdateCollection.newBuilder()
|
||||
.setCollectionName("{collection_name}")
|
||||
.setHnswConfig(HnswConfigDiff.newBuilder().setM(16).build())
|
||||
.build())
|
||||
.get();
|
||||
```
|
||||
|
||||
```csharp
|
||||
using Qdrant.Client;
|
||||
using Qdrant.Client.Grpc;
|
||||
|
||||
var client = new QdrantClient("localhost", 6334);
|
||||
|
||||
await client.UpdateCollectionAsync(
|
||||
collectionName: "{collection_name}",
|
||||
hnswConfig: new HnswConfigDiff { M = 16 }
|
||||
);
|
||||
```
|
||||
|
||||
```go
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/qdrant/go-client/qdrant"
|
||||
)
|
||||
|
||||
qdrant.NewClient(&qdrant.Config{
|
||||
Host: "localhost",
|
||||
Port: 6334,
|
||||
})
|
||||
|
||||
client, err := client.UpdateCollection(context.Background(), &qdrant.UpdateCollection{
|
||||
CollectionName: "{collection_name}",
|
||||
HnswConfig: &qdrant.HnswConfigDiff{
|
||||
M: qdrant.PtrOf(uint64(16)),
|
||||
},
|
||||
})
|
||||
```
|
||||
|
||||
### Disable indexing completely (`indexing_threshold: 0`)
|
||||
|
||||
In case you are doing an initial upload of a large dataset, you might want to disable indexing during upload. It will enable to avoid unnecessary indexing of vectors, which will be overwritten by the next batch.
|
||||
|
||||
Setting `indexing_threshold` to `0` disables indexing altogether:
|
||||
|
||||
```http
|
||||
PUT /collections/{collection_name}
|
||||
{
|
||||
"vectors": {
|
||||
"size": 768,
|
||||
"distance": "Cosine"
|
||||
},
|
||||
"optimizers_config": {
|
||||
"indexing_threshold": 0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
```python
|
||||
from qdrant_client import QdrantClient, models
|
||||
|
||||
client = QdrantClient(url="http://localhost:6333")
|
||||
|
||||
client.create_collection(
|
||||
collection_name="{collection_name}",
|
||||
vectors_config=models.VectorParams(size=768, distance=models.Distance.COSINE),
|
||||
optimizers_config=models.OptimizersConfigDiff(
|
||||
indexing_threshold=0,
|
||||
),
|
||||
)
|
||||
```
|
||||
|
||||
```typescript
|
||||
import { QdrantClient } from "@qdrant/js-client-rest";
|
||||
|
||||
const client = new QdrantClient({ host: "localhost", port: 6333 });
|
||||
|
||||
client.createCollection("{collection_name}", {
|
||||
vectors: {
|
||||
size: 768,
|
||||
distance: "Cosine",
|
||||
},
|
||||
optimizers_config: {
|
||||
indexing_threshold: 0,
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
```rust
|
||||
use qdrant_client::qdrant::{
|
||||
OptimizersConfigDiffBuilder, UpdateCollectionBuilder,
|
||||
};
|
||||
use qdrant_client::Qdrant;
|
||||
|
||||
let client = Qdrant::from_url("http://localhost:6334").build()?;
|
||||
|
||||
client
|
||||
.create_collection(
|
||||
CreateCollectionBuilder::new("{collection_name}")
|
||||
.optimizers_config(OptimizersConfigDiffBuilder::default().indexing_threshold(0)),
|
||||
)
|
||||
.await?;
|
||||
```
|
||||
|
||||
```java
|
||||
import io.qdrant.client.grpc.Collections.CreateCollection;
|
||||
import io.qdrant.client.grpc.Collections.Distance;
|
||||
import io.qdrant.client.grpc.Collections.VectorParams;
|
||||
import io.qdrant.client.grpc.Collections.VectorsConfig;
|
||||
import io.qdrant.client.grpc.Collections.OptimizersConfigDiff;
|
||||
|
||||
QdrantClient client =
|
||||
new QdrantClient(QdrantGrpcClient.newBuilder("localhost", 6334, false).build());
|
||||
|
||||
client.createCollectionAsync(
|
||||
CreateCollection.newBuilder()
|
||||
.setCollectionName("{collection_name}")
|
||||
.setVectorsConfig(
|
||||
VectorsConfig.newBuilder()
|
||||
.setParams(
|
||||
VectorParams.newBuilder()
|
||||
.setSize(768)
|
||||
.setDistance(Distance.Cosine)
|
||||
.build())
|
||||
.build())
|
||||
.setOptimizersConfig(
|
||||
OptimizersConfigDiff.newBuilder()
|
||||
.setIndexingThreshold(0)
|
||||
.build())
|
||||
.build()
|
||||
).get();
|
||||
```
|
||||
|
||||
```csharp
|
||||
using Qdrant.Client;
|
||||
using Qdrant.Client.Grpc;
|
||||
|
||||
var client = new QdrantClient("localhost", 6334);
|
||||
|
||||
await client.CreateCollectionAsync(
|
||||
collectionName: "{collection_name}",
|
||||
vectorsConfig: new VectorParams { Size = 768, Distance = Distance.Cosine },
|
||||
optimizersConfig: new OptimizersConfigDiff { IndexingThreshold = 0 }
|
||||
);
|
||||
```
|
||||
|
||||
```go
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/qdrant/go-client/qdrant"
|
||||
)
|
||||
|
||||
client, err := qdrant.NewClient(&qdrant.Config{
|
||||
Host: "localhost",
|
||||
Port: 6334,
|
||||
})
|
||||
|
||||
client.CreateCollection(context.Background(), &qdrant.CreateCollection{
|
||||
CollectionName: "{collection_name}",
|
||||
VectorsConfig: qdrant.NewVectorsConfig(&qdrant.VectorParams{
|
||||
Size: 768,
|
||||
Distance: qdrant.Distance_Cosine,
|
||||
}),
|
||||
OptimizersConfig: &qdrant.OptimizersConfigDiff{
|
||||
IndexingThreshold: qdrant.PtrOf(uint64(0)),
|
||||
},
|
||||
})
|
||||
```
|
||||
|
||||
<aside role="status">
|
||||
With indexing_threshold set to 0, storage won't be optimized properly, which can lead to high RAM usage as segments accumulate in memory.
|
||||
</aside>
|
||||
|
||||
After upload is done, you can enable indexing by setting `indexing_threshold` to a desired value (default is 10000):
|
||||
|
||||
```http
|
||||
PATCH /collections/{collection_name}
|
||||
{
|
||||
"optimizers_config": {
|
||||
"indexing_threshold": 10000
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
```python
|
||||
from qdrant_client import QdrantClient, models
|
||||
|
||||
client = QdrantClient(url="http://localhost:6333")
|
||||
|
||||
client.update_collection(
|
||||
collection_name="{collection_name}",
|
||||
optimizers_config=models.OptimizersConfigDiff(indexing_threshold=20000),
|
||||
)
|
||||
```
|
||||
|
||||
```typescript
|
||||
import { QdrantClient } from "@qdrant/js-client-rest";
|
||||
|
||||
const client = new QdrantClient({ host: "localhost", port: 6333 });
|
||||
|
||||
client.updateCollection("{collection_name}", {
|
||||
optimizers_config: {
|
||||
indexing_threshold: 10000,
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
```rust
|
||||
use qdrant_client::qdrant::{
|
||||
OptimizersConfigDiffBuilder, UpdateCollectionBuilder,
|
||||
};
|
||||
use qdrant_client::Qdrant;
|
||||
|
||||
let client = Qdrant::from_url("http://localhost:6334").build()?;
|
||||
|
||||
client
|
||||
.update_collection(
|
||||
UpdateCollectionBuilder::new("{collection_name}")
|
||||
.optimizers_config(OptimizersConfigDiffBuilder::default().indexing_threshold(10000)),
|
||||
)
|
||||
.await?;
|
||||
```
|
||||
|
||||
```java
|
||||
import io.qdrant.client.grpc.Collections.UpdateCollection;
|
||||
import io.qdrant.client.grpc.Collections.OptimizersConfigDiff;
|
||||
|
||||
client.updateCollectionAsync(
|
||||
UpdateCollection.newBuilder()
|
||||
.setCollectionName("{collection_name}")
|
||||
.setOptimizersConfig(
|
||||
OptimizersConfigDiff.newBuilder()
|
||||
.setIndexingThreshold(20000)
|
||||
.build()
|
||||
)
|
||||
.build()
|
||||
).get();
|
||||
```
|
||||
|
||||
```csharp
|
||||
using Qdrant.Client;
|
||||
using Qdrant.Client.Grpc;
|
||||
|
||||
var client = new QdrantClient("localhost", 6334);
|
||||
|
||||
await client.UpdateCollectionAsync(
|
||||
collectionName: "{collection_name}",
|
||||
optimizersConfig: new OptimizersConfigDiff { IndexingThreshold = 20000 }
|
||||
);
|
||||
```
|
||||
|
||||
```go
|
||||
import (
|
||||
"context"
|
||||
"github.com/qdrant/go-client/qdrant"
|
||||
)
|
||||
|
||||
client, err := qdrant.NewClient(&qdrant.Config{
|
||||
Host: "localhost",
|
||||
Port: 6334,
|
||||
})
|
||||
|
||||
client.UpdateCollection(context.Background(), &qdrant.UpdateCollection{
|
||||
CollectionName: "{collection_name}",
|
||||
OptimizersConfig: &qdrant.OptimizersConfigDiff{
|
||||
IndexingThreshold: qdrant.PtrOf(uint64(20000)),
|
||||
},
|
||||
})
|
||||
```
|
||||
|
||||
|
||||
|
||||
At this point, Qdrant will begin indexing new and previously unindexed segments in the background.
|
||||
|
||||
## Upload directly to disk
|
||||
## Upload Directly to Disk
|
||||
|
||||
When the vectors you upload do not all fit in RAM, you likely want to use
|
||||
[memmap](/documentation/manage-data/storage/#configuring-memmap-storage)
|
||||
support.
|
||||
|
||||
During collection
|
||||
[creation](/documentation/manage-data/collections/#create-collection),
|
||||
memmaps may be enabled on a per-vector basis using the `on_disk` parameter. This
|
||||
will store vector data directly on disk at all times. It is suitable for
|
||||
ingesting a large amount of data, essential for the billion scale benchmark.
|
||||
During [collection
|
||||
creation](/documentation/manage-data/collections/#create-collection),
|
||||
memmaps can be enabled on a per-vector basis using the `on_disk` parameter. This
|
||||
will store vector data directly on disk at all times.
|
||||
|
||||
Using `memmap_threshold` is not recommended in this case. It would require
|
||||
Using `memmap_threshold` is not recommended in this case. This requires
|
||||
the [optimizer](/documentation/ops-optimization/optimizer/) to constantly
|
||||
transform in-memory segments into memmap segments on disk. This process is
|
||||
slower, and the optimizer can be a bottleneck when ingesting a large amount of
|
||||
data.
|
||||
|
||||
Read more about this in
|
||||
[Configuring Memmap Storage](/documentation/manage-data/storage/#configuring-memmap-storage).
|
||||
|
||||
## Parallel upload into multiple shards
|
||||
|
||||
In Qdrant, each collection is split into shards. Each shard has a separate Write-Ahead-Log (WAL), which is responsible for ordering operations.
|
||||
By creating multiple shards, you can parallelize upload of a large dataset. From 2 to 4 shards per one machine is a reasonable number.
|
||||
|
||||
```http
|
||||
PUT /collections/{collection_name}
|
||||
{
|
||||
"vectors": {
|
||||
"size": 768,
|
||||
"distance": "Cosine"
|
||||
},
|
||||
"shard_number": 2
|
||||
}
|
||||
```
|
||||
|
||||
```python
|
||||
from qdrant_client import QdrantClient, models
|
||||
|
||||
client = QdrantClient(url="http://localhost:6333")
|
||||
|
||||
client.create_collection(
|
||||
collection_name="{collection_name}",
|
||||
vectors_config=models.VectorParams(size=768, distance=models.Distance.COSINE),
|
||||
shard_number=2,
|
||||
)
|
||||
```
|
||||
|
||||
```typescript
|
||||
import { QdrantClient } from "@qdrant/js-client-rest";
|
||||
|
||||
const client = new QdrantClient({ host: "localhost", port: 6333 });
|
||||
|
||||
client.createCollection("{collection_name}", {
|
||||
vectors: {
|
||||
size: 768,
|
||||
distance: "Cosine",
|
||||
},
|
||||
shard_number: 2,
|
||||
});
|
||||
```
|
||||
|
||||
```rust
|
||||
use qdrant_client::qdrant::{CreateCollectionBuilder, Distance, VectorParamsBuilder};
|
||||
use qdrant_client::Qdrant;
|
||||
|
||||
let client = Qdrant::from_url("http://localhost:6334").build()?;
|
||||
|
||||
client
|
||||
.create_collection(
|
||||
CreateCollectionBuilder::new("{collection_name}")
|
||||
.vectors_config(VectorParamsBuilder::new(768, Distance::Cosine))
|
||||
.shard_number(2),
|
||||
)
|
||||
.await?;
|
||||
```
|
||||
|
||||
```java
|
||||
import io.qdrant.client.QdrantClient;
|
||||
import io.qdrant.client.QdrantGrpcClient;
|
||||
import io.qdrant.client.grpc.Collections.CreateCollection;
|
||||
import io.qdrant.client.grpc.Collections.Distance;
|
||||
import io.qdrant.client.grpc.Collections.VectorParams;
|
||||
import io.qdrant.client.grpc.Collections.VectorsConfig;
|
||||
|
||||
QdrantClient client =
|
||||
new QdrantClient(QdrantGrpcClient.newBuilder("localhost", 6334, false).build());
|
||||
|
||||
client
|
||||
.createCollectionAsync(
|
||||
CreateCollection.newBuilder()
|
||||
.setCollectionName("{collection_name}")
|
||||
.setVectorsConfig(
|
||||
VectorsConfig.newBuilder()
|
||||
.setParams(
|
||||
VectorParams.newBuilder()
|
||||
.setSize(768)
|
||||
.setDistance(Distance.Cosine)
|
||||
.build())
|
||||
.build())
|
||||
.setShardNumber(2)
|
||||
.build())
|
||||
.get();
|
||||
```
|
||||
|
||||
```csharp
|
||||
using Qdrant.Client;
|
||||
using Qdrant.Client.Grpc;
|
||||
|
||||
var client = new QdrantClient("localhost", 6334);
|
||||
|
||||
await client.CreateCollectionAsync(
|
||||
collectionName: "{collection_name}",
|
||||
vectorsConfig: new VectorParams { Size = 768, Distance = Distance.Cosine },
|
||||
shardNumber: 2
|
||||
);
|
||||
```
|
||||
|
||||
```go
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/qdrant/go-client/qdrant"
|
||||
)
|
||||
|
||||
client, err := qdrant.NewClient(&qdrant.Config{
|
||||
Host: "localhost",
|
||||
Port: 6334,
|
||||
})
|
||||
|
||||
client.CreateCollection(context.Background(), &qdrant.CreateCollection{
|
||||
CollectionName: "{collection_name}",
|
||||
VectorsConfig: qdrant.NewVectorsConfig(&qdrant.VectorParams{
|
||||
Size: 768,
|
||||
Distance: qdrant.Distance_Cosine,
|
||||
}),
|
||||
ShardNumber: qdrant.PtrOf(uint32(2)),
|
||||
})
|
||||
```
|
||||
For full configuration details, see [Configuring Memmap Storage](/documentation/manage-data/storage/#configuring-memmap-storage).
|
||||
|
||||
Reference in New Issue
Block a user