From cf4119fa3f059ce211d58fdf3fd9ef2acad33a2c Mon Sep 17 00:00:00 2001 From: Abdon Pijpelink Date: Tue, 30 Jun 2026 13:07:43 +0200 Subject: [PATCH] Add subsection about stable ordering to pagination section (#2463) * Add 'Stable Ordering' section to 'Pagination' section * Add FAQ entry * Small edits to the Pagination section * Apply title case to all headers on page * Small edit --- .../documentation/faq/qdrant-fundamentals.md | 12 +++++ .../with-exact-search/_description.md | 1 + .../query-points/with-exact-search/csharp.cs | 17 +++++++ .../with-exact-search/generated/csharp.md | 11 ++++ .../with-exact-search/generated/go.md | 15 ++++++ .../with-exact-search/generated/java.md | 17 +++++++ .../with-exact-search/generated/python.md | 10 ++++ .../with-exact-search/generated/rust.md | 13 +++++ .../with-exact-search/generated/typescript.md | 9 ++++ .../query-points/with-exact-search/go.go | 26 ++++++++++ .../query-points/with-exact-search/http.md | 10 ++++ .../query-points/with-exact-search/java.java | 26 ++++++++++ .../query-points/with-exact-search/python.py | 10 ++++ .../query-points/with-exact-search/rust.rs | 17 +++++++ .../with-exact-search/typescript.ts | 11 ++++ .../_description.md | 1 + .../with-id-exclusion-pagination/csharp.cs | 20 ++++++++ .../generated/csharp.md | 14 +++++ .../generated/go.md | 25 +++++++++ .../generated/java.md | 25 +++++++++ .../generated/python.md | 18 +++++++ .../generated/rust.md | 15 ++++++ .../generated/typescript.md | 15 ++++++ .../with-id-exclusion-pagination/go.go | 36 +++++++++++++ .../with-id-exclusion-pagination/http.md | 12 +++++ .../with-id-exclusion-pagination/java.java | 34 +++++++++++++ .../with-id-exclusion-pagination/python.py | 18 +++++++ .../with-id-exclusion-pagination/rust.rs | 19 +++++++ .../typescript.ts | 17 +++++++ .../query-points/with-offset/csharp.cs | 4 +- .../with-offset/generated/csharp.md | 4 -- .../query-points/with-offset/generated/go.md | 6 +-- .../with-offset/generated/java.md | 3 -- .../with-offset/generated/python.md | 4 -- .../with-offset/generated/rust.md | 2 - .../with-offset/generated/typescript.md | 4 -- .../snippets/query-points/with-offset/go.go | 5 +- .../query-points/with-offset/java.java | 2 + .../query-points/with-offset/python.py | 4 +- .../snippets/query-points/with-offset/rust.rs | 2 +- .../query-points/with-offset/typescript.ts | 4 +- .../content/documentation/search/search.md | 51 ++++++++++++++----- 42 files changed, 526 insertions(+), 43 deletions(-) create mode 100644 qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/_description.md create mode 100644 qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/csharp.cs create mode 100644 qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/generated/csharp.md create mode 100644 qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/generated/go.md create mode 100644 qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/generated/java.md create mode 100644 qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/generated/python.md create mode 100644 qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/generated/rust.md create mode 100644 qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/generated/typescript.md create mode 100644 qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/go.go create mode 100644 qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/http.md create mode 100644 qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/java.java create mode 100644 qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/python.py create mode 100644 qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/rust.rs create mode 100644 qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/typescript.ts create mode 100644 qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/_description.md create mode 100644 qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/csharp.cs create mode 100644 qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/generated/csharp.md create mode 100644 qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/generated/go.md create mode 100644 qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/generated/java.md create mode 100644 qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/generated/python.md create mode 100644 qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/generated/rust.md create mode 100644 qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/generated/typescript.md create mode 100644 qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/go.go create mode 100644 qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/http.md create mode 100644 qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/java.java create mode 100644 qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/python.py create mode 100644 qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/rust.rs create mode 100644 qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/typescript.ts diff --git a/qdrant-landing/content/documentation/faq/qdrant-fundamentals.md b/qdrant-landing/content/documentation/faq/qdrant-fundamentals.md index b90b5f965..d54418dfc 100644 --- a/qdrant-landing/content/documentation/faq/qdrant-fundamentals.md +++ b/qdrant-landing/content/documentation/faq/qdrant-fundamentals.md @@ -183,6 +183,18 @@ Results are generally expected to be consistent for the overlapping portion. How The time value is in seconds and represents the total duration the Qdrant server spent processing the request. It does not include network round-trip time between the client and the server. +### Why do I get duplicate results when paginating through search results? + +Because HNSW is an approximate algorithm, the ranking of results can shift slightly between requests. As a result, paginating with `offset` can return the same point on multiple pages or skip points entirely. This is expected behavior, not a bug. + +There are three ways to work around this: + +- **Client-side pagination** — retrieve a large batch in a single request (for example, the top 100 results) and paginate through it on the client. This avoids multiple round-trips and guarantees no duplicates, at the cost of returning more data than the user sees at once. +- **Exact search** — use exact searches to bypass HNSW and scan all vectors, returning results in a stable, deterministic order. This ensures offset-based pagination works correctly. This is practical only for small collections due to higher latency. +- **Exclude seen IDs** — on each subsequent page, pass a `must_not: has_id` filter containing all point IDs from previous pages. The exclusion list grows by `limit` entries per page, so this works well for sequential, forward-only pagination but isn't practical for jumping to an arbitrary page. + +See also: [Stable Ordering](/documentation/search/search/#stable-ordering) + ### If `limit` is higher than `hnsw_ef`, does Qdrant automatically adjust `hnsw_ef`? Yes. Qdrant internally sets `ef = max(ef, limit)` so that the candidate list is always at least as large as the requested result count. diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/_description.md b/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/_description.md new file mode 100644 index 000000000..80f48a744 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/_description.md @@ -0,0 +1 @@ +This code snippet demonstrates how to run an exact nearest neighbor search by setting the `exact` parameter to `true`. Unlike the default approximate HNSW search, exact search scans all vectors and returns results in a stable, deterministic order. \ No newline at end of file diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/csharp.cs b/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/csharp.cs new file mode 100644 index 000000000..9b8c18be8 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/csharp.cs @@ -0,0 +1,17 @@ +using Qdrant.Client; +using Qdrant.Client.Grpc; + +public class Snippet +{ + public static async Task Run() + { + var client = new QdrantClient("localhost", 6334); // @hide + + await client.QueryAsync( + collectionName: "{collection_name}", + query: new float[] { 0.2f, 0.1f, 0.9f, 0.7f }, + searchParams: new SearchParams { Exact = true }, + limit: 10 + ); + } +} diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/generated/csharp.md b/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/generated/csharp.md new file mode 100644 index 000000000..70fd3e84c --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/generated/csharp.md @@ -0,0 +1,11 @@ +```csharp +using Qdrant.Client; +using Qdrant.Client.Grpc; + +await client.QueryAsync( + collectionName: "{collection_name}", + query: new float[] { 0.2f, 0.1f, 0.9f, 0.7f }, + searchParams: new SearchParams { Exact = true }, + limit: 10 +); +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/generated/go.md b/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/generated/go.md new file mode 100644 index 000000000..ef1f7798f --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/generated/go.md @@ -0,0 +1,15 @@ +```go +import ( + "context" + + "github.com/qdrant/go-client/qdrant" +) + +client.Query(context.Background(), &qdrant.QueryPoints{ + CollectionName: "{collection_name}", + Query: qdrant.NewQuery(0.2, 0.1, 0.9, 0.7), + Params: &qdrant.SearchParams{ + Exact: qdrant.PtrOf(true), + }, +}) +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/generated/java.md b/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/generated/java.md new file mode 100644 index 000000000..75dc3bcd1 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/generated/java.md @@ -0,0 +1,17 @@ +```java +import static io.qdrant.client.QueryFactory.nearest; + +import io.qdrant.client.QdrantClient; +import io.qdrant.client.QdrantGrpcClient; +import io.qdrant.client.grpc.Points.QueryPoints; +import io.qdrant.client.grpc.Points.SearchParams; + +client.queryAsync( + QueryPoints.newBuilder() + .setCollectionName("{collection_name}") + .setQuery(nearest(0.2f, 0.1f, 0.9f, 0.7f)) + .setParams(SearchParams.newBuilder().setExact(true).build()) + .setLimit(10) + .build()) + .get(); +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/generated/python.md b/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/generated/python.md new file mode 100644 index 000000000..a772bfca7 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/generated/python.md @@ -0,0 +1,10 @@ +```python +from qdrant_client import QdrantClient, models + +client.query_points( + collection_name="{collection_name}", + query=[0.2, 0.1, 0.9, 0.7], + search_params=models.SearchParams(exact=True), + limit=10, +) +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/generated/rust.md b/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/generated/rust.md new file mode 100644 index 000000000..3f8550b7a --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/generated/rust.md @@ -0,0 +1,13 @@ +```rust +use qdrant_client::qdrant::{QueryPointsBuilder, SearchParamsBuilder}; +use qdrant_client::Qdrant; + +client + .query( + QueryPointsBuilder::new("{collection_name}") + .query(vec![0.2, 0.1, 0.9, 0.7]) + .limit(10) + .params(SearchParamsBuilder::default().exact(true)), + ) + .await?; +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/generated/typescript.md b/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/generated/typescript.md new file mode 100644 index 000000000..0426a2b71 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/generated/typescript.md @@ -0,0 +1,9 @@ +```typescript +client.query("{collection_name}", { + query: [0.2, 0.1, 0.9, 0.7], + params: { + exact: true, + }, + limit: 10, +}); +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/go.go b/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/go.go new file mode 100644 index 000000000..665c98ae5 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/go.go @@ -0,0 +1,26 @@ +package snippet + +import ( + "context" + + "github.com/qdrant/go-client/qdrant" +) + +func Main() { + // @hide-start + client, err := qdrant.NewClient(&qdrant.Config{ + Host: "localhost", + Port: 6334, + }) + + if err != nil { panic(err) } + // @hide-end + + client.Query(context.Background(), &qdrant.QueryPoints{ + CollectionName: "{collection_name}", + Query: qdrant.NewQuery(0.2, 0.1, 0.9, 0.7), + Params: &qdrant.SearchParams{ + Exact: qdrant.PtrOf(true), + }, + }) +} diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/http.md b/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/http.md new file mode 100644 index 000000000..e0726c0a3 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/http.md @@ -0,0 +1,10 @@ +```http +POST /collections/{collection_name}/points/query +{ + "query": [0.2, 0.1, 0.9, 0.7], + "params": { + "exact": true + }, + "limit": 10 +} +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/java.java b/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/java.java new file mode 100644 index 000000000..448184eed --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/java.java @@ -0,0 +1,26 @@ +package com.example.snippets_amalgamation; + +import static io.qdrant.client.QueryFactory.nearest; + +import io.qdrant.client.QdrantClient; +import io.qdrant.client.QdrantGrpcClient; +import io.qdrant.client.grpc.Points.QueryPoints; +import io.qdrant.client.grpc.Points.SearchParams; + +public class Snippet { + public static void run() throws Exception { + // @hide-start + QdrantClient client = + new QdrantClient(QdrantGrpcClient.newBuilder("localhost", 6334, false).build()); + // @hide-end + + client.queryAsync( + QueryPoints.newBuilder() + .setCollectionName("{collection_name}") + .setQuery(nearest(0.2f, 0.1f, 0.9f, 0.7f)) + .setParams(SearchParams.newBuilder().setExact(true).build()) + .setLimit(10) + .build()) + .get(); + } +} diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/python.py b/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/python.py new file mode 100644 index 000000000..17a6f6aa7 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/python.py @@ -0,0 +1,10 @@ +from qdrant_client import QdrantClient, models + +client = QdrantClient(url="http://localhost:6333") # @hide + +client.query_points( + collection_name="{collection_name}", + query=[0.2, 0.1, 0.9, 0.7], + search_params=models.SearchParams(exact=True), + limit=10, +) diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/rust.rs b/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/rust.rs new file mode 100644 index 000000000..4dd1d4f49 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/rust.rs @@ -0,0 +1,17 @@ +use qdrant_client::qdrant::{QueryPointsBuilder, SearchParamsBuilder}; +use qdrant_client::Qdrant; + +pub async fn main() -> anyhow::Result<()> { + let client = Qdrant::from_url("http://localhost:6334").build()?; // @hide + + client + .query( + QueryPointsBuilder::new("{collection_name}") + .query(vec![0.2, 0.1, 0.9, 0.7]) + .limit(10) + .params(SearchParamsBuilder::default().exact(true)), + ) + .await?; + + Ok(()) +} diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/typescript.ts b/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/typescript.ts new file mode 100644 index 000000000..228af7d70 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-exact-search/typescript.ts @@ -0,0 +1,11 @@ +import { QdrantClient } from "@qdrant/js-client-rest"; // @hide + +const client = new QdrantClient({ host: "localhost", port: 6333 }); // @hide + +client.query("{collection_name}", { + query: [0.2, 0.1, 0.9, 0.7], + params: { + exact: true, + }, + limit: 10, +}); diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/_description.md b/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/_description.md new file mode 100644 index 000000000..9ec416946 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/_description.md @@ -0,0 +1 @@ +This code snippet demonstrates how to paginate search results without duplicate points. By collecting the point IDs returned on each page and passing them to a `must_not: has_id` filter on the next request, each subsequent page excludes all previously seen results. \ No newline at end of file diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/csharp.cs b/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/csharp.cs new file mode 100644 index 000000000..16cc41375 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/csharp.cs @@ -0,0 +1,20 @@ +using Qdrant.Client; +using static Qdrant.Client.Grpc.Conditions; + +public class Snippet +{ + public static async Task Run() + { + var client = new QdrantClient("localhost", 6334); // @hide + + ulong[] seenIds = [83461, 19284, 57392, 44017, 91825]; // IDs returned on previous pages + + // The ! operator negates the condition (must not) + await client.QueryAsync( + collectionName: "{collection_name}", + query: new float[] { 0.2f, 0.1f, 0.9f, 0.7f }, + filter: !HasId(seenIds), + limit: 5 + ); + } +} diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/generated/csharp.md b/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/generated/csharp.md new file mode 100644 index 000000000..21adc3d7a --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/generated/csharp.md @@ -0,0 +1,14 @@ +```csharp +using Qdrant.Client; +using static Qdrant.Client.Grpc.Conditions; + +ulong[] seenIds = [83461, 19284, 57392, 44017, 91825]; // IDs returned on previous pages + +// The ! operator negates the condition (must not) +await client.QueryAsync( + collectionName: "{collection_name}", + query: new float[] { 0.2f, 0.1f, 0.9f, 0.7f }, + filter: !HasId(seenIds), + limit: 5 +); +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/generated/go.md b/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/generated/go.md new file mode 100644 index 000000000..642ea1b69 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/generated/go.md @@ -0,0 +1,25 @@ +```go +import ( + "context" + + "github.com/qdrant/go-client/qdrant" +) + +seenIds := []uint64{83461, 19284, 57392, 44017, 91825} // IDs returned on previous pages + +pointIds := make([]*qdrant.PointId, len(seenIds)) +for i, id := range seenIds { + pointIds[i] = qdrant.NewIDNum(id) +} + +client.Query(context.Background(), &qdrant.QueryPoints{ + CollectionName: "{collection_name}", + Query: qdrant.NewQuery(0.2, 0.1, 0.9, 0.7), + Filter: &qdrant.Filter{ + MustNot: []*qdrant.Condition{ + qdrant.NewHasID(pointIds...), + }, + }, + Limit: qdrant.PtrOf(uint64(5)), +}) +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/generated/java.md b/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/generated/java.md new file mode 100644 index 000000000..d655f9edd --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/generated/java.md @@ -0,0 +1,25 @@ +```java +import static io.qdrant.client.ConditionFactory.hasId; +import static io.qdrant.client.PointIdFactory.id; +import static io.qdrant.client.QueryFactory.nearest; + +import io.qdrant.client.QdrantClient; +import io.qdrant.client.QdrantGrpcClient; +import io.qdrant.client.grpc.Common.Filter; +import io.qdrant.client.grpc.Points.QueryPoints; +import java.util.List; + +var seenIds = List.of(id(83461), id(19284), id(57392), id(44017), id(91825)); // IDs returned on previous pages + +client.queryAsync( + QueryPoints.newBuilder() + .setCollectionName("{collection_name}") + .setQuery(nearest(0.2f, 0.1f, 0.9f, 0.7f)) + .setFilter( + Filter.newBuilder() + .addMustNot(hasId(seenIds)) + .build()) + .setLimit(5) + .build()) + .get(); +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/generated/python.md b/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/generated/python.md new file mode 100644 index 000000000..ebbbcfa4f --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/generated/python.md @@ -0,0 +1,18 @@ +```python +from uuid import UUID + +from qdrant_client import QdrantClient, models + +seen_ids: list[int | str | UUID] = [83461, 19284, 57392, 44017, 91825] # IDs returned on previous pages + +client.query_points( + collection_name="{collection_name}", + query=[0.2, 0.1, 0.9, 0.7], + query_filter=models.Filter( + must_not=[ + models.HasIdCondition(has_id=seen_ids), + ] + ), + limit=5, +) +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/generated/rust.md b/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/generated/rust.md new file mode 100644 index 000000000..c5b7b5a2a --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/generated/rust.md @@ -0,0 +1,15 @@ +```rust +use qdrant_client::qdrant::{Condition, Filter, QueryPointsBuilder}; +use qdrant_client::Qdrant; + +let seen_ids = vec![83461u64, 19284, 57392, 44017, 91825]; // IDs returned on previous pages + +client + .query( + QueryPointsBuilder::new("{collection_name}") + .query(vec![0.2, 0.1, 0.9, 0.7]) + .filter(Filter::must_not([Condition::has_id(seen_ids)])) + .limit(5), + ) + .await?; +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/generated/typescript.md b/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/generated/typescript.md new file mode 100644 index 000000000..668e6923f --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/generated/typescript.md @@ -0,0 +1,15 @@ +```typescript +const seenIds = [83461, 19284, 57392, 44017, 91825]; // IDs returned on previous pages + +client.query("{collection_name}", { + query: [0.2, 0.1, 0.9, 0.7], + filter: { + must_not: [ + { + has_id: seenIds, + }, + ], + }, + limit: 5, +}); +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/go.go b/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/go.go new file mode 100644 index 000000000..37f09f26d --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/go.go @@ -0,0 +1,36 @@ +package snippet + +import ( + "context" + + "github.com/qdrant/go-client/qdrant" +) + +func Main() { + // @hide-start + client, err := qdrant.NewClient(&qdrant.Config{ + Host: "localhost", + Port: 6334, + }) + + if err != nil { panic(err) } + // @hide-end + + seenIds := []uint64{83461, 19284, 57392, 44017, 91825} // IDs returned on previous pages + + pointIds := make([]*qdrant.PointId, len(seenIds)) + for i, id := range seenIds { + pointIds[i] = qdrant.NewIDNum(id) + } + + client.Query(context.Background(), &qdrant.QueryPoints{ + CollectionName: "{collection_name}", + Query: qdrant.NewQuery(0.2, 0.1, 0.9, 0.7), + Filter: &qdrant.Filter{ + MustNot: []*qdrant.Condition{ + qdrant.NewHasID(pointIds...), + }, + }, + Limit: qdrant.PtrOf(uint64(5)), + }) +} diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/http.md b/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/http.md new file mode 100644 index 000000000..c5bc01ad0 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/http.md @@ -0,0 +1,12 @@ +```http +POST /collections/{collection_name}/points/query +{ + "query": [0.2, 0.1, 0.9, 0.7], + "filter": { + "must_not": [ + { "has_id": [83461, 19284, 57392, 44017, 91825] } + ] + }, + "limit": 5 +} +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/java.java b/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/java.java new file mode 100644 index 000000000..3c722b87a --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/java.java @@ -0,0 +1,34 @@ +package com.example.snippets_amalgamation; + +import static io.qdrant.client.ConditionFactory.hasId; +import static io.qdrant.client.PointIdFactory.id; +import static io.qdrant.client.QueryFactory.nearest; + +import io.qdrant.client.QdrantClient; +import io.qdrant.client.QdrantGrpcClient; +import io.qdrant.client.grpc.Common.Filter; +import io.qdrant.client.grpc.Points.QueryPoints; +import java.util.List; + +public class Snippet { + public static void run() throws Exception { + // @hide-start + QdrantClient client = + new QdrantClient(QdrantGrpcClient.newBuilder("localhost", 6334, false).build()); + // @hide-end + + var seenIds = List.of(id(83461), id(19284), id(57392), id(44017), id(91825)); // IDs returned on previous pages + + client.queryAsync( + QueryPoints.newBuilder() + .setCollectionName("{collection_name}") + .setQuery(nearest(0.2f, 0.1f, 0.9f, 0.7f)) + .setFilter( + Filter.newBuilder() + .addMustNot(hasId(seenIds)) + .build()) + .setLimit(5) + .build()) + .get(); + } +} diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/python.py b/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/python.py new file mode 100644 index 000000000..ac07af014 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/python.py @@ -0,0 +1,18 @@ +from uuid import UUID + +from qdrant_client import QdrantClient, models + +client = QdrantClient(url="http://localhost:6333") # @hide + +seen_ids: list[int | str | UUID] = [83461, 19284, 57392, 44017, 91825] # IDs returned on previous pages + +client.query_points( + collection_name="{collection_name}", + query=[0.2, 0.1, 0.9, 0.7], + query_filter=models.Filter( + must_not=[ + models.HasIdCondition(has_id=seen_ids), + ] + ), + limit=5, +) diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/rust.rs b/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/rust.rs new file mode 100644 index 000000000..b68f7271d --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/rust.rs @@ -0,0 +1,19 @@ +use qdrant_client::qdrant::{Condition, Filter, QueryPointsBuilder}; +use qdrant_client::Qdrant; + +pub async fn main() -> anyhow::Result<()> { + let client = Qdrant::from_url("http://localhost:6334").build()?; // @hide + + let seen_ids = vec![83461u64, 19284, 57392, 44017, 91825]; // IDs returned on previous pages + + client + .query( + QueryPointsBuilder::new("{collection_name}") + .query(vec![0.2, 0.1, 0.9, 0.7]) + .filter(Filter::must_not([Condition::has_id(seen_ids)])) + .limit(5), + ) + .await?; + + Ok(()) +} diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/typescript.ts b/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/typescript.ts new file mode 100644 index 000000000..cdcdd0a81 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-id-exclusion-pagination/typescript.ts @@ -0,0 +1,17 @@ +import { QdrantClient } from "@qdrant/js-client-rest"; // @hide + +const client = new QdrantClient({ host: "localhost", port: 6333 }); // @hide + +const seenIds = [83461, 19284, 57392, 44017, 91825]; // IDs returned on previous pages + +client.query("{collection_name}", { + query: [0.2, 0.1, 0.9, 0.7], + filter: { + must_not: [ + { + has_id: seenIds, + }, + ], + }, + limit: 5, +}); diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/csharp.cs b/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/csharp.cs index 4ef98e58a..a682d7281 100644 --- a/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/csharp.cs +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/csharp.cs @@ -1,10 +1,10 @@ -using Qdrant.Client; +using Qdrant.Client; // @hide public class Snippet { public static async Task Run() { - var client = new QdrantClient("localhost", 6334); + var client = new QdrantClient("localhost", 6334); // @hide await client.QueryAsync( collectionName: "{collection_name}", diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/generated/csharp.md b/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/generated/csharp.md index fd71b4d92..5316176e7 100644 --- a/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/generated/csharp.md +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/generated/csharp.md @@ -1,8 +1,4 @@ ```csharp -using Qdrant.Client; - -var client = new QdrantClient("localhost", 6334); - await client.QueryAsync( collectionName: "{collection_name}", query: new float[] { 0.2f, 0.1f, 0.9f, 0.7f }, diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/generated/go.md b/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/generated/go.md index 88bf5eac9..829bd2b4b 100644 --- a/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/generated/go.md +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/generated/go.md @@ -5,16 +5,12 @@ import ( "github.com/qdrant/go-client/qdrant" ) -client, err := qdrant.NewClient(&qdrant.Config{ - Host: "localhost", - Port: 6334, -}) - client.Query(context.Background(), &qdrant.QueryPoints{ CollectionName: "{collection_name}", Query: qdrant.NewQuery(0.2, 0.1, 0.9, 0.7), WithPayload: qdrant.NewWithPayload(true), WithVectors: qdrant.NewWithVectors(true), + Limit: qdrant.PtrOf(uint64(10)), Offset: qdrant.PtrOf(uint64(100)), }) ``` diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/generated/java.md b/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/generated/java.md index 986688ea7..29a0920e3 100644 --- a/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/generated/java.md +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/generated/java.md @@ -8,9 +8,6 @@ import io.qdrant.client.WithVectorsSelectorFactory; import io.qdrant.client.grpc.Points.QueryPoints; import java.util.List; -QdrantClient client = - new QdrantClient(QdrantGrpcClient.newBuilder("localhost", 6334, false).build()); - client.queryAsync( QueryPoints.newBuilder() .setCollectionName("{collection_name}") diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/generated/python.md b/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/generated/python.md index c372451dd..b41e1d774 100644 --- a/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/generated/python.md +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/generated/python.md @@ -1,8 +1,4 @@ ```python -from qdrant_client import QdrantClient - -client = QdrantClient(url="http://localhost:6333") - client.query_points( collection_name="{collection_name}", query=[0.2, 0.1, 0.9, 0.7], diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/generated/rust.md b/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/generated/rust.md index 76986e3bc..dffa07920 100644 --- a/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/generated/rust.md +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/generated/rust.md @@ -2,8 +2,6 @@ use qdrant_client::qdrant::QueryPointsBuilder; use qdrant_client::Qdrant; -let client = Qdrant::from_url("http://localhost:6334").build()?; - client .query( QueryPointsBuilder::new("{collection_name}") diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/generated/typescript.md b/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/generated/typescript.md index fdaccfc03..d7db00867 100644 --- a/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/generated/typescript.md +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/generated/typescript.md @@ -1,8 +1,4 @@ ```typescript -import { QdrantClient } from "@qdrant/js-client-rest"; - -const client = new QdrantClient({ host: "localhost", port: 6333 }); - client.query("{collection_name}", { query: [0.2, 0.1, 0.9, 0.7], with_vector: true, diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/go.go b/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/go.go index 3e3b3232e..1bcd65303 100644 --- a/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/go.go +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/go.go @@ -7,18 +7,21 @@ import ( ) func Main() { + // @hide-start client, err := qdrant.NewClient(&qdrant.Config{ Host: "localhost", Port: 6334, }) - if err != nil { panic(err) } // @hide + if err != nil { panic(err) } + // @hide-end client.Query(context.Background(), &qdrant.QueryPoints{ CollectionName: "{collection_name}", Query: qdrant.NewQuery(0.2, 0.1, 0.9, 0.7), WithPayload: qdrant.NewWithPayload(true), WithVectors: qdrant.NewWithVectors(true), + Limit: qdrant.PtrOf(uint64(10)), Offset: qdrant.PtrOf(uint64(100)), }) } diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/java.java b/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/java.java index e3b106297..2e63c81fc 100644 --- a/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/java.java +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/java.java @@ -11,8 +11,10 @@ import java.util.List; public class Snippet { public static void run() throws Exception { + // @hide-start QdrantClient client = new QdrantClient(QdrantGrpcClient.newBuilder("localhost", 6334, false).build()); + // @hide-end client.queryAsync( QueryPoints.newBuilder() diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/python.py b/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/python.py index bb1b33dff..bfc48584a 100644 --- a/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/python.py +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/python.py @@ -1,6 +1,6 @@ -from qdrant_client import QdrantClient +from qdrant_client import QdrantClient # @hide -client = QdrantClient(url="http://localhost:6333") +client = QdrantClient(url="http://localhost:6333") # @hide client.query_points( collection_name="{collection_name}", diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/rust.rs b/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/rust.rs index 0b510a2c2..5c1f01c20 100644 --- a/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/rust.rs +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/rust.rs @@ -2,7 +2,7 @@ use qdrant_client::qdrant::QueryPointsBuilder; use qdrant_client::Qdrant; pub async fn main() -> anyhow::Result<()> { - let client = Qdrant::from_url("http://localhost:6334").build()?; + let client = Qdrant::from_url("http://localhost:6334").build()?; // @hide client .query( diff --git a/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/typescript.ts b/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/typescript.ts index 31ff105e3..453b46285 100644 --- a/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/typescript.ts +++ b/qdrant-landing/content/documentation/headless/snippets/query-points/with-offset/typescript.ts @@ -1,6 +1,6 @@ -import { QdrantClient } from "@qdrant/js-client-rest"; +import { QdrantClient } from "@qdrant/js-client-rest"; // @hide -const client = new QdrantClient({ host: "localhost", port: 6333 }); +const client = new QdrantClient({ host: "localhost", port: 6333 }); // @hide client.query("{collection_name}", { query: [0.2, 0.1, 0.9, 0.7], diff --git a/qdrant-landing/content/documentation/search/search.md b/qdrant-landing/content/documentation/search/search.md index d9f5af0d1..9b1d44c85 100644 --- a/qdrant-landing/content/documentation/search/search.md +++ b/qdrant-landing/content/documentation/search/search.md @@ -8,7 +8,7 @@ aliases: - /documentation/concepts/search/ --- -# Similarity search +# Similarity Search Searching for the nearest vectors is at the core of many representational learning applications. Modern neural networks are trained to transform objects into vectors so that objects close in the real world appear close in vector space. @@ -142,7 +142,7 @@ In general, the speed of the search is proportional to the number of non-zero va {{< code-snippet path="/documentation/headless/snippets/query-points/sparse-vectors/" >}} -### Filtering results by score +### Filtering Results by Score In addition to payload filtering, it might be useful to filter out results with a low similarity score. For example, if you know the minimal acceptance score for your model and do not want any results which are less similar than the threshold. @@ -151,7 +151,7 @@ It will exclude all results with a score worse than the given. -### Payload and vector in the result +### Payload and Vector in the Result By default, retrieval methods do not return any stored information such as payload and vectors. Additional parameters `with_vectors` and `with_payload` @@ -205,7 +205,7 @@ $$ \text{Estimated filter selectivity} = $$ Since ACORN is significantly slower (approximately 2-10x in typical scenarios) but improves recall for restrictive filters, tuning this parameter is about deciding when the accuracy improvement justifies the performance cost. -## Batch search API +## Batch Search API The batch search API enables to perform multiple search requests via a single request. @@ -266,22 +266,47 @@ collection `another_collection`. ## Pagination -Search and [recommendation](/documentation/search/explore/#recommendation-api) APIs allow to skip first results of the search and return only the result starting from some specified offset: +The Search and [recommendation](/documentation/search/explore/#recommendation-api) APIs allow you to skip the first results and return only the results starting from a specified offset: Example: {{< code-snippet path="/documentation/headless/snippets/query-points/with-offset/" >}} -Is equivalent to retrieving the 11th page with 10 records per page. +This is equivalent to retrieving the 11th page with 10 records per page. -Vector-based retrieval in general and HNSW index in particular, are not designed to be paginated. -It is impossible to retrieve Nth closest vector without retrieving the first N vectors first. +Vector-based retrieval in general, and the HNSW index in particular, are not designed to be paginated. It is impossible to retrieve the Nth closest vector without internally retrieving the first N vectors first. However, using the `offset` parameter saves resources by reducing network traffic and the number of times the storage is accessed. Using the `offset` parameter internally retrieves `offset + limit` points, but only accesses the payload and vector of those points that are actually returned. -However, using the offset parameter saves the resources by reducing network traffic and the number of times the storage is accessed. +### Stable Ordering -Using an `offset` parameter, will require to internally retrieve `offset + limit` points, but only access payload and vector from the storage those points which are going to be actually returned. +Because HNSW search is approximate, the ranking of results can shift slightly between requests. As a result, paginating with `offset` can return the same point on multiple pages or skip points entirely. + +There are several ways to work around this: + +#### Client-Side Pagination + +Retrieve a large batch in a single request and paginate through it on the client. For example, fetch the top 100 results at once and let the user browse them 10 at a time. This avoids multiple round-trips and guarantees no duplicates. + +The trade-off is increased latency, and returning more data than the user actually needs. + +#### Exact Search + +Use exact searches to bypass HNSW and scan all vectors, returning results in a stable, deterministic order. This ensures that offset-based pagination works correctly. + +The trade-off is higher latency, which makes this practical only for small collections. + +{{< code-snippet path="/documentation/headless/snippets/query-points/with-exact-search/" >}} + +#### Exclude Seen IDs + +To avoid duplicates, on subsequent pages, add a `must_not: has_id` filter containing all point IDs collected from previous pages. This excludes all previously seen points from the results: + +{{< code-snippet path="/documentation/headless/snippets/query-points/with-id-exclusion-pagination/" >}} + +Repeat this pattern on every page, expanding the exclusion list with each set of results. + + ## Grouping API @@ -346,7 +371,7 @@ Consider having points with the following payloads: With the ***groups*** API, you will be able to get the best *N* points for each document, assuming that the payload of the points contains the document ID. Of course there will be times where the best *N* points cannot be fulfilled due to lack of points or a big distance with respect to the query. In every case, the `group_size` is a best-effort parameter, akin to the `limit` parameter. -### Search groups +### Search Groups REST API ([Schema](https://api.qdrant.tech/api-reference/search/query-points-groups)): @@ -402,7 +427,7 @@ If the `group_by` field of a point is an array (e.g. `"document_id": ["a", "b"]` * Only [keyword](/documentation/manage-data/payload/#keyword) and [integer](/documentation/manage-data/payload/#integer) payload values are supported for the `group_by` parameter. Payload values with other types will be ignored. * At the moment, pagination is not enabled when using **groups**, so the `offset` parameter is not allowed. -### Lookup in groups +### Lookup in Groups When the points in a group share large fields like titles, abstracts, or full document vectors, copying that data onto every point inflates storage and forces you to rewrite every chunk whenever a shared field changes. @@ -484,7 +509,7 @@ Random sampling API is a part of [Universal Query API](#query-api) and can be us {{< code-snippet path="/documentation/headless/snippets/query-points/random-sample/" >}} -## Query planning +## Query Planning Depending on the filter used in the search - there are several possible scenarios for query execution. Qdrant chooses one of the query execution options depending on the available indexes, the complexity of the conditions and the cardinality of the filtering result.