From 053badb564bf53999d94da02cb7346f3dc7b20a7 Mon Sep 17 00:00:00 2001 From: generall Date: Tue, 13 Aug 2024 18:48:28 +0200 Subject: [PATCH 1/3] on-disk, tenants and principal --- .../documentation/concepts/indexing.md | 375 ++++++++++++++++++ 1 file changed, 375 insertions(+) diff --git a/qdrant-landing/content/documentation/concepts/indexing.md b/qdrant-landing/content/documentation/concepts/indexing.md index 14f5ea658..085969f92 100644 --- a/qdrant-landing/content/documentation/concepts/indexing.md +++ b/qdrant-landing/content/documentation/concepts/indexing.md @@ -420,6 +420,381 @@ await client.CreatePayloadIndexAsync( ); ``` +### On-disk payload index + +*Available as of v1.11.0* + +By default all payload-related structures are stored in memory. In this way vector index can quickly access payload values during search. +As latency in this case is critical, it is recommended to keep hot payload indexes in memory. + +There are, however, cases when payload indexes are too large or rarely used. In those cases, it is possible to store payload indexes on disk. + + + +To configure on-disk payload index, you can use the following index parameters: + +```http +PUT /collections/{collection_name}/index +{ + "field_name": "workspace_2", + "field_schema": { + "type": "keyword", + "on_disk": true + } +} +``` + +```python +client.create_payload_index( + collection_name="{collection_name}", + field_name="workspace_2", + field_schema=models.KeywordIndexParams( + type="keyword", + on_disk=True, + ), +) +``` + +```typescript +client.createPayloadIndex("{collection_name}", { + field_name: "workspace_2", + field_schema: { + type: "keyword", + on_disk: true + }, +}); +``` + +```rust +use qdrant_client::qdrant::{ + CreateFieldIndexCollectionBuilder, + KeywordIndexParamsBuilder, + FieldType +}; +use qdrant_client::{Qdrant, QdrantError}; + +let client = Qdrant::from_url("http://localhost:6334").build()?; + +client.create_field_index( + CreateFieldIndexCollectionBuilder::new( + "{collection_name}", + "workspace_2", + FieldType::Keyword, + ) + .field_index_params( + KeywordIndexParamsBuilder::default() + .on_disk(true), + ), +); +``` + +```java +import io.qdrant.client.QdrantClient; +import io.qdrant.client.QdrantGrpcClient; +import io.qdrant.client.grpc.Collections.PayloadIndexParams; +import io.qdrant.client.grpc.Collections.PayloadSchemaType; +import io.qdrant.client.grpc.Collections.KeywordIndexParams; + +QdrantClient client = + new QdrantClient(QdrantGrpcClient.newBuilder("localhost", 6334, false).build()); + +client + .createPayloadIndexAsync( + "{collection_name}", + "workspace_2", + PayloadSchemaType.Keyword, + PayloadIndexParams.newBuilder() + .setKeywordIndexParams( + KeywordIndexParams.newBuilder() + .setOnDisk(true) + .build()) + .build(), + null, + null, + null) + .get(); +``` + +```csharp +using Qdrant.Client; +using Qdrant.Client.Grpc; + +var client = new QdrantClient("localhost", 6334); + +await client.CreatePayloadIndexAsync( + collectionName: "{collection_name}", + fieldName: "workspace_2", + schemaType: PayloadSchemaType.Keyword, + indexParams: new PayloadIndexParams + { + KeywordIndexParams = new KeywordIndexParams + { + OnDisk = true + } + } +); + +``` + +Payload index on-disk is supported for following types: + +* `keyword` +* `integer` +* `float` +* `datetime` +* `uuid` + +The list will be extended in future versions. + +### Tenant Index + +*Available as of v1.11.0* + +Many vector search use-cases require multi-tenancy support. In multi-tenant scenario the collection is expected to contain multiple subsets of data, each subset belonging to a different tenant. + +Qdrant supports efficient multi-tenant search by enabling [special configuration](../guides/multiple-partitions/) vector index, which disables global search and only builds sub-indexes for each tenant. + + + +But knowing that the collection contains multiple tenants unlocks more opportunities for optimization. +For enabling Qdrant to optimize storage even further, you can enable tenant index for payload fields. + +This option will tell Qdrant which fields are used for tenant identification and will allow Qdrant to structure storage for faster search of tenant-specific data. +One example of such optimization is localizing tenant-specific data closer on disk, which will reduce the number of disk reads during search. + +To enable tenant index for a field, you can use the following index parameters: + +```http +PUT /collections/{collection_name}/index +{ + "field_name": "workspace_2", + "field_schema": { + "type": "keyword", + "is_tenant": true + } +} +``` + +```python +client.create_payload_index( + collection_name="{collection_name}", + field_name="workspace_2", + field_schema=models.KeywordIndexParams( + type="keyword", + is_tenant=True, + ), +) +``` + +```typescript +client.createPayloadIndex("{collection_name}", { + field_name: "workspace_2", + field_schema: { + type: "keyword", + is_tenant: true + }, +}); +``` + +```rust +use qdrant_client::qdrant::{ + CreateFieldIndexCollectionBuilder, + KeywordIndexParamsBuilder, + FieldType +}; +use qdrant_client::{Qdrant, QdrantError}; + +let client = Qdrant::from_url("http://localhost:6334").build()?; + +client.create_field_index( + CreateFieldIndexCollectionBuilder::new( + "{collection_name}", + "workspace_2", + FieldType::Keyword, + ) + .field_index_params( + KeywordIndexParamsBuilder::default() + .is_tenant(true), + ), +); +``` + +```java +import io.qdrant.client.QdrantClient; +import io.qdrant.client.QdrantGrpcClient; +import io.qdrant.client.grpc.Collections.PayloadIndexParams; +import io.qdrant.client.grpc.Collections.PayloadSchemaType; +import io.qdrant.client.grpc.Collections.KeywordIndexParams; + +QdrantClient client = + new QdrantClient(QdrantGrpcClient.newBuilder("localhost", 6334, false).build()); + +client + .createPayloadIndexAsync( + "{collection_name}", + "workspace_2", + PayloadSchemaType.Keyword, + PayloadIndexParams.newBuilder() + .setKeywordIndexParams( + KeywordIndexParams.newBuilder() + .setIsTenant(true) + .build()) + .build(), + null, + null, + null) + .get(); +``` + +```csharp +using Qdrant.Client; +using Qdrant.Client.Grpc; + +var client = new QdrantClient("localhost", 6334); + +await client.CreatePayloadIndexAsync( + collectionName: "{collection_name}", + fieldName: "workspace_2", + schemaType: PayloadSchemaType.Keyword, + indexParams: new PayloadIndexParams + { + KeywordIndexParams = new KeywordIndexParams + { + IsTenant = true + } + } +); + +``` + + +Tenant optimization is supported for following types: + +* `keyword` +* `uuid` + +### Principal Index + +*Available as of v1.11.0* + +Similar to the tenant index, the principal index is used to optimize storage for faster search, assuming that the search request is primarily filtered by the principal field. + +A good example of a use case for the principal index is time-related data, where each point is associated with a timestamp. In this case, the principal index can be used to optimize storage for faster search with time-based filters. + +```http +PUT /collections/{collection_name}/index +{ + "field_name": "timestamp", + "field_schema": { + "type": "integer", + "is_principal": true + } +} +``` + +```python +client.create_payload_index( + collection_name="{collection_name}", + field_name="timestamp", + field_schema=models.KeywordIndexParams( + type="integer", + is_principal=True, + ), +) +``` + +```typescript +client.createPayloadIndex("{collection_name}", { + field_name: "timestamp", + field_schema: { + type: "integer", + is_principal: true + }, +}); +``` + +```rust +use qdrant_client::qdrant::{ + CreateFieldIndexCollectionBuilder, + IntegerdIndexParamsBuilder, + FieldType +}; +use qdrant_client::{Qdrant, QdrantError}; + +let client = Qdrant::from_url("http://localhost:6334").build()?; + +client.create_field_index( + CreateFieldIndexCollectionBuilder::new( + "{collection_name}", + "timestamp", + FieldType::Integer, + ) + .field_index_params( + IntegerdIndexParamsBuilder::default() + .is_principal(true), + ), +); +``` + +```java +import io.qdrant.client.QdrantClient; +import io.qdrant.client.QdrantGrpcClient; +import io.qdrant.client.grpc.Collections.PayloadIndexParams; +import io.qdrant.client.grpc.Collections.PayloadSchemaType; +import io.qdrant.client.grpc.Collections.IntegerIndexParams; + +QdrantClient client = + new QdrantClient(QdrantGrpcClient.newBuilder("localhost", 6334, false).build()); + +client + .createPayloadIndexAsync( + "{collection_name}", + "timestamp", + PayloadSchemaType.Integer, + PayloadIndexParams.newBuilder() + .setIntegerIndexParams( + KeywordIndexParams.newBuilder() + .setIsPrincipa(true) + .build()) + .build(), + null, + null, + null) + .get(); +``` + +```csharp +using Qdrant.Client; +using Qdrant.Client.Grpc; + +var client = new QdrantClient("localhost", 6334); + +await client.CreatePayloadIndexAsync( + collectionName: "{collection_name}", + fieldName: "timestamp", + schemaType: PayloadSchemaType.Integer, + indexParams: new PayloadIndexParams + { + IntegerIndexParams = new IntegerIndexParams + { + IsPrincipal = true + } + } +); + +``` + +Principal optimization is supported for following types: + +* `integer` +* `float` +* `datetime` + + ## Vector Index A vector index is a data structure built on vectors through a specific mathematical model. From 37aa7104093d35cfdcd70cdc577c1dfee75c088b Mon Sep 17 00:00:00 2001 From: generall Date: Tue, 13 Aug 2024 18:54:10 +0200 Subject: [PATCH 2/3] remove workspace_2 --- .../documentation/concepts/indexing.md | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/qdrant-landing/content/documentation/concepts/indexing.md b/qdrant-landing/content/documentation/concepts/indexing.md index 085969f92..39920875c 100644 --- a/qdrant-landing/content/documentation/concepts/indexing.md +++ b/qdrant-landing/content/documentation/concepts/indexing.md @@ -438,7 +438,7 @@ To configure on-disk payload index, you can use the following index parameters: ```http PUT /collections/{collection_name}/index { - "field_name": "workspace_2", + "field_name": "payload_field_name", "field_schema": { "type": "keyword", "on_disk": true @@ -449,7 +449,7 @@ PUT /collections/{collection_name}/index ```python client.create_payload_index( collection_name="{collection_name}", - field_name="workspace_2", + field_name="payload_field_name", field_schema=models.KeywordIndexParams( type="keyword", on_disk=True, @@ -459,7 +459,7 @@ client.create_payload_index( ```typescript client.createPayloadIndex("{collection_name}", { - field_name: "workspace_2", + field_name: "payload_field_name", field_schema: { type: "keyword", on_disk: true @@ -480,7 +480,7 @@ let client = Qdrant::from_url("http://localhost:6334").build()?; client.create_field_index( CreateFieldIndexCollectionBuilder::new( "{collection_name}", - "workspace_2", + "payload_field_name", FieldType::Keyword, ) .field_index_params( @@ -503,7 +503,7 @@ QdrantClient client = client .createPayloadIndexAsync( "{collection_name}", - "workspace_2", + "payload_field_name", PayloadSchemaType.Keyword, PayloadIndexParams.newBuilder() .setKeywordIndexParams( @@ -525,7 +525,7 @@ var client = new QdrantClient("localhost", 6334); await client.CreatePayloadIndexAsync( collectionName: "{collection_name}", - fieldName: "workspace_2", + fieldName: "payload_field_name", schemaType: PayloadSchemaType.Keyword, indexParams: new PayloadIndexParams { @@ -571,7 +571,7 @@ To enable tenant index for a field, you can use the following index parameters: ```http PUT /collections/{collection_name}/index { - "field_name": "workspace_2", + "field_name": "payload_field_name", "field_schema": { "type": "keyword", "is_tenant": true @@ -582,7 +582,7 @@ PUT /collections/{collection_name}/index ```python client.create_payload_index( collection_name="{collection_name}", - field_name="workspace_2", + field_name="payload_field_name", field_schema=models.KeywordIndexParams( type="keyword", is_tenant=True, @@ -592,7 +592,7 @@ client.create_payload_index( ```typescript client.createPayloadIndex("{collection_name}", { - field_name: "workspace_2", + field_name: "payload_field_name", field_schema: { type: "keyword", is_tenant: true @@ -613,7 +613,7 @@ let client = Qdrant::from_url("http://localhost:6334").build()?; client.create_field_index( CreateFieldIndexCollectionBuilder::new( "{collection_name}", - "workspace_2", + "payload_field_name", FieldType::Keyword, ) .field_index_params( @@ -636,7 +636,7 @@ QdrantClient client = client .createPayloadIndexAsync( "{collection_name}", - "workspace_2", + "payload_field_name", PayloadSchemaType.Keyword, PayloadIndexParams.newBuilder() .setKeywordIndexParams( @@ -658,7 +658,7 @@ var client = new QdrantClient("localhost", 6334); await client.CreatePayloadIndexAsync( collectionName: "{collection_name}", - fieldName: "workspace_2", + fieldName: "payload_field_name", schemaType: PayloadSchemaType.Keyword, indexParams: new PayloadIndexParams { From 0ab13c555e1784693e5bb8c3c568e6f79186a11b Mon Sep 17 00:00:00 2001 From: davidmyriel Date: Tue, 13 Aug 2024 18:07:28 -0700 Subject: [PATCH 3/3] fix language --- .../content/documentation/concepts/indexing.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/qdrant-landing/content/documentation/concepts/indexing.md b/qdrant-landing/content/documentation/concepts/indexing.md index 39920875c..8e9f911c6 100644 --- a/qdrant-landing/content/documentation/concepts/indexing.md +++ b/qdrant-landing/content/documentation/concepts/indexing.md @@ -424,7 +424,7 @@ await client.CreatePayloadIndexAsync( *Available as of v1.11.0* -By default all payload-related structures are stored in memory. In this way vector index can quickly access payload values during search. +By default all payload-related structures are stored in memory. In this way, the vector index can quickly access payload values during search. As latency in this case is critical, it is recommended to keep hot payload indexes in memory. There are, however, cases when payload indexes are too large or rarely used. In those cases, it is possible to store payload indexes on disk. @@ -552,7 +552,7 @@ The list will be extended in future versions. *Available as of v1.11.0* -Many vector search use-cases require multi-tenancy support. In multi-tenant scenario the collection is expected to contain multiple subsets of data, each subset belonging to a different tenant. +Many vector search use-cases require multitenancy. In a multi-tenant scenario the collection is expected to contain multiple subsets of data, where each subset belongs to a different tenant. Qdrant supports efficient multi-tenant search by enabling [special configuration](../guides/multiple-partitions/) vector index, which disables global search and only builds sub-indexes for each tenant. @@ -560,8 +560,8 @@ Qdrant supports efficient multi-tenant search by enabling [special configuration In Qdrant, tenants are not necessarily non-overlapping. It is possible to have subsets of data that belong to multiple tenants. -But knowing that the collection contains multiple tenants unlocks more opportunities for optimization. -For enabling Qdrant to optimize storage even further, you can enable tenant index for payload fields. +However, knowing that the collection contains multiple tenants unlocks more opportunities for optimization. +To optimize storage in Qdrant further, you can enable tenant indexing for payload fields. This option will tell Qdrant which fields are used for tenant identification and will allow Qdrant to structure storage for faster search of tenant-specific data. One example of such optimization is localizing tenant-specific data closer on disk, which will reduce the number of disk reads during search. @@ -672,7 +672,7 @@ await client.CreatePayloadIndexAsync( ``` -Tenant optimization is supported for following types: +Tenant optimization is supported for the following datatypes: * `keyword` * `uuid`