Merge pull request #1527 from qdrant/upt-bulk-upload-recs

[Docs] Updating recommendations for bulk upload
2026-09-25 14:08:30 +02:00 · 2025-03-28 13:56:22 -03:00
parent b241a1528e 1bb980e517
commit 886452e0f9
1 changed files with 487 additions and 4 deletions
@@ -16,12 +16,264 @@ We recommend using our [Rust client library](https://github.com/qdrant/rust-clie

 If you are not using Rust, you might want to consider parallelizing your upload process.

-## Disable indexing during upload
+## Choose an Indexing Strategy

-In case you are doing an initial upload of a large dataset, you might want to disable indexing during upload.
-It will enable to avoid unnecessary indexing of vectors, which will be overwritten by the next batch.
+Qdrant incrementally builds an HNSW index for dense vectors as new data arrives. This ensures fast search, but indexing is memory- and CPU-intensive. During bulk ingestion, frequent index updates can reduce throughput and increase resource usage.

-To disable indexing during upload, set `indexing_threshold` to `0`:
+To control this behavior and optimize for your system’s limits, adjust the following parameters:
+
+| Your Goal                                 | What to Do                                      | Configuration                                      |
+|-------------------------------------------|-------------------------------------------------|----------------------------------------------------|
+| Fastest upload, tolerate high RAM usage           | Disable indexing completely                     | `indexing_threshold: 0`                           |
+| Low memory usage during upload            | Defer HNSW graph construction (recommended)                  | `m: 0`                 |
+| Faster index availability after upload       | Keep indexing enabled (default behavior)        | `m: 16`, `indexing_threshold: 20000` *(default)* |
+
+Indexing must be re-enabled after upload to activate fast HNSW search if it was disabled during ingestion.
+
+
+### Defer HNSW graph construction (`m: 0`)
+
+For dense vectors, setting the HNSW `m` parameter to `0` disables index building entirely. Vectors will still be stored, but not indexed until you enable indexing later.
+
+```http
+PUT /collections/{collection_name}
+{
+    "vectors": {
+      "size": 768,
+      "distance": "Cosine"
+    },
+    "hnsw_config": {
+        "m": 0
+    }
+}
+```
+
+```python
+from qdrant_client import QdrantClient, models
+
+client = QdrantClient(url="http://localhost:6333")
+
+client.create_collection(
+    collection_name="{collection_name}",
+    vectors_config=models.VectorParams(size=768, distance=models.Distance.COSINE),
+    hnsw_config=models.HnswConfigDiff(
+        m=0,
+    ),
+)
+```
+
+```typescript
+import { QdrantClient } from "@qdrant/js-client-rest";
+
+const client = new QdrantClient({ host: "localhost", port: 6333 });
+
+client.createCollection("{collection_name}", {
+  vectors: {
+    size: 768,
+    distance: "Cosine",
+  },
+  hnsw_config: {
+    m: 0,
+  },
+});
+```
+
+```rust
+use qdrant_client::qdrant::{
+    CreateCollectionBuilder, Distance, HnswConfigDiffBuilder, VectorParamsBuilder,
+};
+use qdrant_client::Qdrant;
+
+let client = Qdrant::from_url("http://localhost:6334").build()?;
+
+client
+    .create_collection(
+        CreateCollectionBuilder::new("{collection_name}")
+            .vectors_config(VectorParamsBuilder::new(768, Distance::Cosine))
+            .hnsw_config(HnswConfigDiffBuilder::default().m(0)),
+    )
+    .await?;
+```
+
+```java
+import io.qdrant.client.QdrantClient;
+import io.qdrant.client.QdrantGrpcClient;
+import io.qdrant.client.grpc.Collections.CreateCollection;
+import io.qdrant.client.grpc.Collections.Distance;
+import io.qdrant.client.grpc.Collections.HnswConfigDiff;
+import io.qdrant.client.grpc.Collections.VectorParams;
+import io.qdrant.client.grpc.Collections.VectorsConfig;
+
+QdrantClient client =
+    new QdrantClient(QdrantGrpcClient.newBuilder("localhost", 6334, false).build());
+
+client
+    .createCollectionAsync(
+        CreateCollection.newBuilder()
+            .setCollectionName("{collection_name}")
+            .setVectorsConfig(
+                VectorsConfig.newBuilder()
+                    .setParams(
+                        VectorParams.newBuilder()
+                            .setSize(768)
+                            .setDistance(Distance.Cosine)
+                            .build())
+                    .build())
+            .setHnswConfig(HnswConfigDiff.newBuilder().setM(0).build())
+            .build())
+    .get();
+```
+
+```csharp
+using Qdrant.Client;
+using Qdrant.Client.Grpc;
+
+var client = new QdrantClient("localhost", 6334);
+
+await client.CreateCollectionAsync(
+	collectionName: "{collection_name}",
+	vectorsConfig: new VectorParams { Size = 768, Distance = Distance.Cosine },
+	hnswConfig: new HnswConfigDiff { M = 0 }
+);
+```
+
+```go
+import (
+	"context"
+
+	"github.com/qdrant/go-client/qdrant"
+)
+
+client, err := qdrant.NewClient(&qdrant.Config{
+	Host: "localhost",
+	Port: 6334,
+})
+
+client.CreateCollection(context.Background(), &qdrant.CreateCollection{
+	CollectionName: "{collection_name}",
+	VectorsConfig: qdrant.NewVectorsConfig(&qdrant.VectorParams{
+		Size:     768,
+		Distance: qdrant.Distance_Cosine,
+	}),
+	HnswConfig: &qdrant.HnswConfigDiff{
+		M:        qdrant.PtrOf(uint64(0)),
+	},
+})
+```
+
+Once ingestion is complete, re-enable HNSW by setting `m` to your production value (usually 16 or 32).
+
+```http
+PATCH /collections/{collection_name}
+{
+    "vectors": {
+      "size": 768,
+      "distance": "Cosine"
+    },
+    "hnsw_config": {
+        "m": 16
+    }
+}
+```
+
+```python
+from qdrant_client import QdrantClient, models
+
+client = QdrantClient(url="http://localhost:6333")
+
+client.update_collection(
+    collection_name="{collection_name}",
+    vectors_config=models.VectorParams(size=768, distance=models.Distance.COSINE),
+    hnsw_config=models.HnswConfigDiff(
+        m=16,
+    ),
+)
+```
+
+```typescript
+import { QdrantClient } from "@qdrant/js-client-rest";
+
+const client = new QdrantClient({ host: "localhost", port: 6333 });
+
+client.updateCollection("{collection_name}", {
+  vectors: {
+    size: 768,
+    distance: "Cosine",
+  },
+  hnsw_config: {
+    m: 16,
+  },
+});
+```
+
+```rust
+use qdrant_client::qdrant::{
+    UpdateCollectionBuilder, HnswConfigDiffBuilder,
+};
+use qdrant_client::Qdrant;
+
+let client = Qdrant::from_url("http://localhost:6334").build()?;
+
+client
+    .update_collection(
+        UpdateCollectionBuilder::new("{collection_name}")
+            .hnsw_config(HnswConfigDiffBuilder::default().m(16)),
+    )
+    .await?;
+```
+
+```java
+import io.qdrant.client.grpc.Collections.UpdateCollection;
+import io.qdrant.client.grpc.Collections.HnswConfigDiff;
+
+QdrantClient client =
+    new QdrantClient(QdrantGrpcClient.newBuilder("localhost", 6334, false).build());
+
+client.updateCollectionAsync(
+    UpdateCollection.newBuilder()
+        .setCollectionName("{collection_name}")
+        .setHnswConfig(HnswConfigDiff.newBuilder().setM(16).build())
+        .build())
+    .get();
+```
+
+```csharp
+using Qdrant.Client;
+using Qdrant.Client.Grpc;
+
+var client = new QdrantClient("localhost", 6334);
+
+await client.UpdateCollectionAsync(
+	collectionName: "{collection_name}",
+	hnswConfig: new HnswConfigDiff { M = 16 }
+);
+```
+
+```go
+import (
+	"context"
+
+	"github.com/qdrant/go-client/qdrant"
+)
+
+qdrant.NewClient(&qdrant.Config{
+	Host: "localhost",
+	Port: 6334,
+})
+
+client, err := client.UpdateCollection(context.Background(), &qdrant.UpdateCollection{
+	CollectionName: "{collection_name}",
+	HnswConfig: &qdrant.HnswConfigDiff{
+		M:        qdrant.PtrOf(uint64(16)),
+	},
+})
+```
+
+### Disable indexing completely (`indexing_threshold: 0`)
+
+In case you are doing an initial upload of a large dataset, you might want to disable indexing during upload. It will enable to avoid unnecessary indexing of vectors, which will be overwritten by the next batch.
+
+Setting `indexing_threshold` to `0` disables indexing altogether:

 ```http
 PUT /collections/{collection_name}
@@ -66,6 +318,92 @@ client.createCollection("{collection_name}", {
 });
 ```

+```rust
+use qdrant_client::qdrant::{
+    OptimizersConfigDiffBuilder, UpdateCollectionBuilder,
+};
+use qdrant_client::Qdrant;
+
+let client = Qdrant::from_url("http://localhost:6334").build()?;
+
+client
+    .create_collection(
+        CreateCollectionBuilder::new("{collection_name}")
+            .optimizers_config(OptimizersConfigDiffBuilder::default().indexing_threshold(0)),
+    )
+    .await?;
+```
+
+```java
+import io.qdrant.client.grpc.Collections.CreateCollection;
+import io.qdrant.client.grpc.Collections.Distance;
+import io.qdrant.client.grpc.Collections.VectorParams;
+import io.qdrant.client.grpc.Collections.VectorsConfig;
+import io.qdrant.client.grpc.Collections.OptimizersConfigDiff;
+
+QdrantClient client =
+    new QdrantClient(QdrantGrpcClient.newBuilder("localhost", 6334, false).build());
+
+client.createCollectionAsync(
+    CreateCollection.newBuilder()
+        .setCollectionName("{collection_name}")
+        .setVectorsConfig(
+            VectorsConfig.newBuilder()
+                .setParams(
+                    VectorParams.newBuilder()
+                        .setSize(768)
+                        .setDistance(Distance.Cosine)
+                        .build())
+                .build())
+        .setOptimizersConfig(
+            OptimizersConfigDiff.newBuilder()
+                .setIndexingThreshold(0)
+                .build())
+        .build()
+).get();
+```
+
+```csharp
+using Qdrant.Client;
+using Qdrant.Client.Grpc;
+
+var client = new QdrantClient("localhost", 6334);
+
+await client.CreateCollectionAsync(
+    collectionName: "{collection_name}",
+    vectorsConfig: new VectorParams { Size = 768, Distance = Distance.Cosine },
+    optimizersConfig: new OptimizersConfigDiff { IndexingThreshold = 0 }
+);
+```
+
+```go
+import (
+	"context"
+
+	"github.com/qdrant/go-client/qdrant"
+)
+
+client, err := qdrant.NewClient(&qdrant.Config{
+	Host: "localhost",
+	Port: 6334,
+})
+
+client.CreateCollection(context.Background(), &qdrant.CreateCollection{
+	CollectionName: "{collection_name}",
+	VectorsConfig: qdrant.NewVectorsConfig(&qdrant.VectorParams{
+		Size:     768,
+		Distance: qdrant.Distance_Cosine,
+	}),
+	OptimizersConfig: &qdrant.OptimizersConfigDiff{
+		IndexingThreshold: qdrant.PtrOf(uint64(0)),
+	},
+})
+```
+
+<aside role="status">
+With indexing_threshold set to 0, storage won't be optimized properly, which can lead to high RAM usage as segments accumulate in memory.
+</aside>
+
 After upload is done, you can enable indexing by setting `indexing_threshold` to a desired value (default is 20000):

 ```http
@@ -100,6 +438,73 @@ client.updateCollection("{collection_name}", {
 });
 ```

+```rust
+use qdrant_client::qdrant::{
+    OptimizersConfigDiffBuilder, UpdateCollectionBuilder,
+};
+use qdrant_client::Qdrant;
+
+let client = Qdrant::from_url("http://localhost:6334").build()?;
+
+client
+    .update_collection(
+        UpdateCollectionBuilder::new("{collection_name}")
+            .optimizers_config(OptimizersConfigDiffBuilder::default().indexing_threshold(20000)),
+    )
+    .await?;
+```
+
+```java
+import io.qdrant.client.grpc.Collections.UpdateCollection;
+import io.qdrant.client.grpc.Collections.OptimizersConfigDiff;
+
+client.updateCollectionAsync(
+    UpdateCollection.newBuilder()
+        .setCollectionName("{collection_name}")
+        .setOptimizersConfig(
+            OptimizersConfigDiff.newBuilder()
+                .setIndexingThreshold(20000)
+                .build()
+        )
+        .build()
+).get();
+```
+
+```csharp
+using Qdrant.Client;
+using Qdrant.Client.Grpc;
+
+var client = new QdrantClient("localhost", 6334);
+
+await client.UpdateCollectionAsync(
+    collectionName: "{collection_name}",
+    optimizersConfig: new OptimizersConfigDiff { IndexingThreshold = 20000 }
+);
+```
+
+```go
+import (
+	"context"
+	"github.com/qdrant/go-client/qdrant"
+)
+
+client, err := qdrant.NewClient(&qdrant.Config{
+	Host: "localhost",
+	Port: 6334,
+})
+
+client.UpdateCollection(context.Background(), &qdrant.UpdateCollection{
+	CollectionName: "{collection_name}",
+	OptimizersConfig: &qdrant.OptimizersConfigDiff{
+		IndexingThreshold: qdrant.PtrOf(uint64(20000)),
+	},
+})
+```
+
+
+
+At this point, Qdrant will begin indexing new and previously unindexed segments in the background.
+
 ## Upload directly to disk

 When the vectors you upload do not all fit in RAM, you likely want to use
@@ -162,3 +567,81 @@ client.createCollection("{collection_name}", {
  shard_number: 2,
 });
 ```
+
+```rust
+use qdrant_client::qdrant::{CreateCollectionBuilder, Distance, VectorParamsBuilder};
+use qdrant_client::Qdrant;
+
+let client = Qdrant::from_url("http://localhost:6334").build()?;
+
+client
+    .create_collection(
+        CreateCollectionBuilder::new("{collection_name}")
+            .vectors_config(VectorParamsBuilder::new(768, Distance::Cosine))
+            .shard_number(2),
+    )
+    .await?;
+```
+
+```java
+import io.qdrant.client.QdrantClient;
+import io.qdrant.client.QdrantGrpcClient;
+import io.qdrant.client.grpc.Collections.CreateCollection;
+import io.qdrant.client.grpc.Collections.Distance;
+import io.qdrant.client.grpc.Collections.VectorParams;
+import io.qdrant.client.grpc.Collections.VectorsConfig;
+
+QdrantClient client =
+    new QdrantClient(QdrantGrpcClient.newBuilder("localhost", 6334, false).build());
+
+client
+    .createCollectionAsync(
+        CreateCollection.newBuilder()
+            .setCollectionName("{collection_name}")
+            .setVectorsConfig(
+                VectorsConfig.newBuilder()
+                    .setParams(
+                        VectorParams.newBuilder()
+                            .setSize(768)
+                            .setDistance(Distance.Cosine)
+                            .build())
+                    .build())
+            .setShardNumber(2)
+            .build())
+    .get();
+```
+
+```csharp
+using Qdrant.Client;
+using Qdrant.Client.Grpc;
+
+var client = new QdrantClient("localhost", 6334);
+
+await client.CreateCollectionAsync(
+	collectionName: "{collection_name}",
+	vectorsConfig: new VectorParams { Size = 768, Distance = Distance.Cosine },
+	shardNumber: 2
+);
+```
+
+```go
+import (
+	"context"
+
+	"github.com/qdrant/go-client/qdrant"
+)
+
+client, err := qdrant.NewClient(&qdrant.Config{
+	Host: "localhost",
+	Port: 6334,
+})
+
+client.CreateCollection(context.Background(), &qdrant.CreateCollection{
+	CollectionName: "{collection_name}",
+	VectorsConfig: qdrant.NewVectorsConfig(&qdrant.VectorParams{
+		Size:     768,
+		Distance: qdrant.Distance_Cosine,
+	}),
+	ShardNumber: qdrant.PtrOf(uint32(2)),
+})
+```