mirror of
https://github.com/qdrant/landing_page.git
synced 2026-10-02 01:18:30 +02:00
Include imports in Edge Rust snippets (#2437)
* Allow rust code snippets to start with a comment line * Include imports in Edge Rust snippets * Add import per code block; use fqn when class name is ambiguous
This commit is contained in:
@@ -67,7 +67,7 @@ class LanguageRust(Language):
|
|||||||
|
|
||||||
RE_CODE = re.compile(
|
RE_CODE = re.compile(
|
||||||
r"""
|
r"""
|
||||||
(?P<uses> (?:use\s[^;]+;\n|\n)* )
|
(?P<uses> (?:use\s[^;]+;\n|//[^\n]*\n|\n)* )
|
||||||
pub\s+async\s+fn\s+main\(\)\s*->\s*anyhow::Result<\(\)>\s*\{\n
|
pub\s+async\s+fn\s+main\(\)\s*->\s*anyhow::Result<\(\)>\s*\{\n
|
||||||
(?P<body> .*? )
|
(?P<body> .*? )
|
||||||
\s*Ok\(\(\)\)\n
|
\s*Ok\(\(\)\)\n
|
||||||
|
|||||||
@@ -18,16 +18,12 @@ Instead of starting with an empty Edge Shard, you may want to initialize it with
|
|||||||
|
|
||||||
When creating a snapshot for synchronization, specify the applicable server-side shard ID in the snapshot URL. This allows for a single collection to serve multiple independent users or devices, each with its own Edge Shard. Read more about Qdrant's sharding strategy in the [Tiered Multitenancy Documentation](/documentation/manage-data/multitenancy/#tiered-multitenancy).
|
When creating a snapshot for synchronization, specify the applicable server-side shard ID in the snapshot URL. This allows for a single collection to serve multiple independent users or devices, each with its own Edge Shard. Read more about Qdrant's sharding strategy in the [Tiered Multitenancy Documentation](/documentation/manage-data/multitenancy/#tiered-multitenancy).
|
||||||
|
|
||||||
First, craft a snapshot URL:
|
First, craft a snapshot URL and use the URL to download the snapshot to the local disk. Next, use the snapshot to initialize a new Edge Shard.
|
||||||
|
|
||||||
{{< code-snippet path="/documentation/headless/snippets/edge/synchronization-patterns/" block="snapshot-url" >}}
|
|
||||||
|
|
||||||
Note that this example uses shard ID `0`.
|
|
||||||
|
|
||||||
Using the snapshot URL, you can download the snapshot to the local disk and use its data to initialize a new Edge Shard.
|
|
||||||
|
|
||||||
{{< code-snippet path="/documentation/headless/snippets/edge/synchronization-patterns/" block="restore-snapshot" >}}
|
{{< code-snippet path="/documentation/headless/snippets/edge/synchronization-patterns/" block="restore-snapshot" >}}
|
||||||
|
|
||||||
|
Note that this example uses shard ID `0`.
|
||||||
|
|
||||||
This code first downloads the snapshot to a temporary directory. Next, `EdgeShard.unpack_snapshot` unpacks the downloaded snapshot into the data directory, and an EdgeShard is initialized using the unpacked data and configuration.
|
This code first downloads the snapshot to a temporary directory. Next, `EdgeShard.unpack_snapshot` unpacks the downloaded snapshot into the data directory, and an EdgeShard is initialized using the unpacked data and configuration.
|
||||||
|
|
||||||
The `edge_shard` will use the same configuration and the same file structure as the source collection from which the snapshot was created, including vector and payload indexes.
|
The `edge_shard` will use the same configuration and the same file structure as the source collection from which the snapshot was created, including vector and payload indexes.
|
||||||
|
|||||||
+5
@@ -1,4 +1,9 @@
|
|||||||
```rust
|
```rust
|
||||||
|
const SHARD_DIRECTORY: &str = "./qdrant-edge-bm25";
|
||||||
|
|
||||||
|
use std::path::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let config = EdgeConfigBuilder::new()
|
let config = EdgeConfigBuilder::new()
|
||||||
.sparse_vector("text", EdgeSparseVectorParamsBuilder::new()
|
.sparse_vector("text", EdgeSparseVectorParamsBuilder::new()
|
||||||
.modifier(Modifier::Idf)
|
.modifier(Modifier::Idf)
|
||||||
|
|||||||
+2
@@ -1,4 +1,6 @@
|
|||||||
```rust
|
```rust
|
||||||
|
use qdrant_edge::bm25_embed::*;
|
||||||
|
|
||||||
let bm25 = EdgeBm25::new(EdgeBm25Config {
|
let bm25 = EdgeBm25::new(EdgeBm25Config {
|
||||||
language: Some("english".to_string()),
|
language: Some("english".to_string()),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
|
|||||||
+3
@@ -1,4 +1,7 @@
|
|||||||
```rust
|
```rust
|
||||||
|
use qdrant_edge::external::serde_json::json;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let docs = [
|
let docs = [
|
||||||
(1u64, "the quick brown fox", "Article 1"),
|
(1u64, "the quick brown fox", "Article 1"),
|
||||||
(2, "a lazy dog sleeps", "Article 2"),
|
(2, "a lazy dog sleeps", "Article 2"),
|
||||||
|
|||||||
+2
@@ -1,4 +1,6 @@
|
|||||||
```rust
|
```rust
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let query_vector = bm25.embed_query("clever fox");
|
let query_vector = bm25.embed_query("clever fox");
|
||||||
|
|
||||||
let results = shard.query(QueryRequest {
|
let results = shard.query(QueryRequest {
|
||||||
|
|||||||
+10
-11
@@ -1,17 +1,9 @@
|
|||||||
```rust
|
```rust
|
||||||
use std::path::Path;
|
|
||||||
|
|
||||||
use qdrant_edge::bm25_embed::{EdgeBm25, EdgeBm25Config};
|
|
||||||
use qdrant_edge::external::serde_json::json;
|
|
||||||
use qdrant_edge::{
|
|
||||||
EdgeConfigBuilder, EdgeShard, EdgeSparseVectorParamsBuilder, Modifier,
|
|
||||||
NamedQuery, PointInsertOperations, PointOperations, PointStruct,
|
|
||||||
QueryEnum, QueryRequest, ScoringQuery, UpdateOperation,
|
|
||||||
VectorInternal, Vectors, WithPayloadInterface, WithVector,
|
|
||||||
};
|
|
||||||
|
|
||||||
const SHARD_DIRECTORY: &str = "./qdrant-edge-bm25";
|
const SHARD_DIRECTORY: &str = "./qdrant-edge-bm25";
|
||||||
|
|
||||||
|
use std::path::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let config = EdgeConfigBuilder::new()
|
let config = EdgeConfigBuilder::new()
|
||||||
.sparse_vector("text", EdgeSparseVectorParamsBuilder::new()
|
.sparse_vector("text", EdgeSparseVectorParamsBuilder::new()
|
||||||
.modifier(Modifier::Idf)
|
.modifier(Modifier::Idf)
|
||||||
@@ -20,11 +12,16 @@ let config = EdgeConfigBuilder::new()
|
|||||||
|
|
||||||
let shard = EdgeShard::new(Path::new(SHARD_DIRECTORY), config)?;
|
let shard = EdgeShard::new(Path::new(SHARD_DIRECTORY), config)?;
|
||||||
|
|
||||||
|
use qdrant_edge::bm25_embed::*;
|
||||||
|
|
||||||
let bm25 = EdgeBm25::new(EdgeBm25Config {
|
let bm25 = EdgeBm25::new(EdgeBm25Config {
|
||||||
language: Some("english".to_string()),
|
language: Some("english".to_string()),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
|
use qdrant_edge::external::serde_json::json;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let docs = [
|
let docs = [
|
||||||
(1u64, "the quick brown fox", "Article 1"),
|
(1u64, "the quick brown fox", "Article 1"),
|
||||||
(2, "a lazy dog sleeps", "Article 2"),
|
(2, "a lazy dog sleeps", "Article 2"),
|
||||||
@@ -44,6 +41,8 @@ shard.update(UpdateOperation::PointOperation(
|
|||||||
))?;
|
))?;
|
||||||
shard.optimize()?;
|
shard.optimize()?;
|
||||||
|
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let query_vector = bm25.embed_query("clever fox");
|
let query_vector = bm25.embed_query("clever fox");
|
||||||
|
|
||||||
let results = shard.query(QueryRequest {
|
let results = shard.query(QueryRequest {
|
||||||
|
|||||||
@@ -1,19 +1,11 @@
|
|||||||
use std::path::Path;
|
// @block-start configure-bm25-shard
|
||||||
|
|
||||||
use qdrant_edge::bm25_embed::{EdgeBm25, EdgeBm25Config};
|
|
||||||
use qdrant_edge::external::serde_json::json;
|
|
||||||
use qdrant_edge::{
|
|
||||||
EdgeConfigBuilder, EdgeShard, EdgeSparseVectorParamsBuilder, Modifier,
|
|
||||||
NamedQuery, PointInsertOperations, PointOperations, PointStruct,
|
|
||||||
QueryEnum, QueryRequest, ScoringQuery, UpdateOperation,
|
|
||||||
VectorInternal, Vectors, WithPayloadInterface, WithVector,
|
|
||||||
};
|
|
||||||
|
|
||||||
pub async fn main() -> anyhow::Result<()> {
|
pub async fn main() -> anyhow::Result<()> {
|
||||||
const SHARD_DIRECTORY: &str = "./qdrant-edge-bm25";
|
const SHARD_DIRECTORY: &str = "./qdrant-edge-bm25";
|
||||||
fs_err::create_dir_all(SHARD_DIRECTORY)?; // @hide
|
fs_err::create_dir_all(SHARD_DIRECTORY)?; // @hide
|
||||||
|
|
||||||
// @block-start configure-bm25-shard
|
use std::path::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let config = EdgeConfigBuilder::new()
|
let config = EdgeConfigBuilder::new()
|
||||||
.sparse_vector("text", EdgeSparseVectorParamsBuilder::new()
|
.sparse_vector("text", EdgeSparseVectorParamsBuilder::new()
|
||||||
.modifier(Modifier::Idf)
|
.modifier(Modifier::Idf)
|
||||||
@@ -24,6 +16,8 @@ pub async fn main() -> anyhow::Result<()> {
|
|||||||
// @block-end configure-bm25-shard
|
// @block-end configure-bm25-shard
|
||||||
|
|
||||||
// @block-start create-bm25
|
// @block-start create-bm25
|
||||||
|
use qdrant_edge::bm25_embed::*;
|
||||||
|
|
||||||
let bm25 = EdgeBm25::new(EdgeBm25Config {
|
let bm25 = EdgeBm25::new(EdgeBm25Config {
|
||||||
language: Some("english".to_string()),
|
language: Some("english".to_string()),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
@@ -31,6 +25,9 @@ pub async fn main() -> anyhow::Result<()> {
|
|||||||
// @block-end create-bm25
|
// @block-end create-bm25
|
||||||
|
|
||||||
// @block-start embed-and-upsert
|
// @block-start embed-and-upsert
|
||||||
|
use qdrant_edge::external::serde_json::json;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let docs = [
|
let docs = [
|
||||||
(1u64, "the quick brown fox", "Article 1"),
|
(1u64, "the quick brown fox", "Article 1"),
|
||||||
(2, "a lazy dog sleeps", "Article 2"),
|
(2, "a lazy dog sleeps", "Article 2"),
|
||||||
@@ -52,6 +49,8 @@ pub async fn main() -> anyhow::Result<()> {
|
|||||||
// @block-end embed-and-upsert
|
// @block-end embed-and-upsert
|
||||||
|
|
||||||
// @block-start query-with-bm25
|
// @block-start query-with-bm25
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let query_vector = bm25.embed_query("clever fox");
|
let query_vector = bm25.embed_query("clever fox");
|
||||||
|
|
||||||
let results = shard.query(QueryRequest {
|
let results = shard.query(QueryRequest {
|
||||||
|
|||||||
+2
@@ -1,4 +1,6 @@
|
|||||||
```rust
|
```rust
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
const VECTOR_NAME: &str = "my-vector";
|
const VECTOR_NAME: &str = "my-vector";
|
||||||
const VECTOR_DIMENSION: usize = 4;
|
const VECTOR_DIMENSION: usize = 4;
|
||||||
|
|
||||||
|
|||||||
+2
@@ -1,4 +1,6 @@
|
|||||||
```rust
|
```rust
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let config = EdgeConfigBuilder::new()
|
let config = EdgeConfigBuilder::new()
|
||||||
.on_disk_payload(true)
|
.on_disk_payload(true)
|
||||||
.vector(
|
.vector(
|
||||||
|
|||||||
+2
@@ -1,4 +1,6 @@
|
|||||||
```rust
|
```rust
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
edge_shard.update(UpdateOperation::FieldIndexOperation(
|
edge_shard.update(UpdateOperation::FieldIndexOperation(
|
||||||
FieldIndexOperations::CreateIndex(CreateIndex {
|
FieldIndexOperations::CreateIndex(CreateIndex {
|
||||||
field_name: "color".try_into().unwrap(),
|
field_name: "color".try_into().unwrap(),
|
||||||
|
|||||||
+2
@@ -1,4 +1,6 @@
|
|||||||
```rust
|
```rust
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let facet_response = edge_shard.facet(FacetRequest {
|
let facet_response = edge_shard.facet(FacetRequest {
|
||||||
key: "color".try_into().unwrap(),
|
key: "color".try_into().unwrap(),
|
||||||
limit: 10,
|
limit: 10,
|
||||||
|
|||||||
+2
@@ -1,4 +1,6 @@
|
|||||||
```rust
|
```rust
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let filter = Filter {
|
let filter = Filter {
|
||||||
should: None,
|
should: None,
|
||||||
min_should: None,
|
min_should: None,
|
||||||
|
|||||||
+3
@@ -1,4 +1,7 @@
|
|||||||
```rust
|
```rust
|
||||||
|
use std::path::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let edge_shard = EdgeShard::new(
|
let edge_shard = EdgeShard::new(
|
||||||
Path::new(SHARD_DIRECTORY),
|
Path::new(SHARD_DIRECTORY),
|
||||||
config,
|
config,
|
||||||
|
|||||||
+3
@@ -1,3 +1,6 @@
|
|||||||
```rust
|
```rust
|
||||||
|
use std::path::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let edge_shard = EdgeShard::load(Path::new(SHARD_DIRECTORY), None)?;
|
let edge_shard = EdgeShard::load(Path::new(SHARD_DIRECTORY), None)?;
|
||||||
```
|
```
|
||||||
|
|||||||
+2
@@ -1,4 +1,6 @@
|
|||||||
```rust
|
```rust
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
edge_shard.update(UpdateOperation::VectorNameOperation(
|
edge_shard.update(UpdateOperation::VectorNameOperation(
|
||||||
VectorNameOperations::CreateVectorName(CreateVectorName {
|
VectorNameOperations::CreateVectorName(CreateVectorName {
|
||||||
vector_name: "text".to_string(),
|
vector_name: "text".to_string(),
|
||||||
|
|||||||
+2
@@ -1,4 +1,6 @@
|
|||||||
```rust
|
```rust
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let results = edge_shard.query(QueryRequest {
|
let results = edge_shard.query(QueryRequest {
|
||||||
prefetches: vec![],
|
prefetches: vec![],
|
||||||
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery {
|
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery {
|
||||||
|
|||||||
+2
@@ -1,4 +1,6 @@
|
|||||||
```rust
|
```rust
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let retrieved = edge_shard.retrieve(
|
let retrieved = edge_shard.retrieve(
|
||||||
&[PointId::NumId(1)],
|
&[PointId::NumId(1)],
|
||||||
Some(WithPayloadInterface::Bool(true)),
|
Some(WithPayloadInterface::Bool(true)),
|
||||||
|
|||||||
+28
-15
@@ -1,23 +1,10 @@
|
|||||||
```rust
|
```rust
|
||||||
use std::path::Path;
|
|
||||||
|
|
||||||
use qdrant_edge::EdgeShard;
|
|
||||||
use qdrant_edge::{
|
|
||||||
Condition, CreateIndex, CreateVectorName, Distance, EdgeConfigBuilder,
|
|
||||||
EdgeOptimizersConfig, EdgeVectorParamsBuilder, FacetRequest, FieldCondition,
|
|
||||||
FieldIndexOperations, Filter, Match, MatchValue, Modifier, NamedQuery,
|
|
||||||
PayloadFieldSchema, PayloadSchemaType, PointId, PointInsertOperations,
|
|
||||||
PointOperations, PointStruct, PointStructPersisted, QueryEnum, QueryRequest,
|
|
||||||
ScoringQuery, SparseVectorConfig, UpdateOperation, ValueVariants,
|
|
||||||
VectorNameConfig, VectorNameOperations, Vectors, WalOptions,
|
|
||||||
WithPayloadInterface, WithVector,
|
|
||||||
};
|
|
||||||
use serde_json::json;
|
|
||||||
|
|
||||||
const SHARD_DIRECTORY: &str = "./qdrant-edge-directory";
|
const SHARD_DIRECTORY: &str = "./qdrant-edge-directory";
|
||||||
|
|
||||||
fs_err::create_dir_all(SHARD_DIRECTORY)?;
|
fs_err::create_dir_all(SHARD_DIRECTORY)?;
|
||||||
|
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
const VECTOR_NAME: &str = "my-vector";
|
const VECTOR_NAME: &str = "my-vector";
|
||||||
const VECTOR_DIMENSION: usize = 4;
|
const VECTOR_DIMENSION: usize = 4;
|
||||||
|
|
||||||
@@ -31,11 +18,17 @@ let config = EdgeConfigBuilder::new()
|
|||||||
)
|
)
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
|
use std::path::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let edge_shard = EdgeShard::new(
|
let edge_shard = EdgeShard::new(
|
||||||
Path::new(SHARD_DIRECTORY),
|
Path::new(SHARD_DIRECTORY),
|
||||||
config,
|
config,
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
|
use serde_json::json;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let points: Vec<PointStructPersisted> = vec![
|
let points: Vec<PointStructPersisted> = vec![
|
||||||
PointStruct::new(
|
PointStruct::new(
|
||||||
1u64,
|
1u64,
|
||||||
@@ -51,12 +44,16 @@ edge_shard.update(UpdateOperation::PointOperation(
|
|||||||
),
|
),
|
||||||
))?;
|
))?;
|
||||||
|
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let retrieved = edge_shard.retrieve(
|
let retrieved = edge_shard.retrieve(
|
||||||
&[PointId::NumId(1)],
|
&[PointId::NumId(1)],
|
||||||
Some(WithPayloadInterface::Bool(true)),
|
Some(WithPayloadInterface::Bool(true)),
|
||||||
Some(WithVector::Bool(false)),
|
Some(WithVector::Bool(false)),
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
edge_shard.update(UpdateOperation::VectorNameOperation(
|
edge_shard.update(UpdateOperation::VectorNameOperation(
|
||||||
VectorNameOperations::CreateVectorName(CreateVectorName {
|
VectorNameOperations::CreateVectorName(CreateVectorName {
|
||||||
vector_name: "text".to_string(),
|
vector_name: "text".to_string(),
|
||||||
@@ -67,6 +64,8 @@ edge_shard.update(UpdateOperation::VectorNameOperation(
|
|||||||
}),
|
}),
|
||||||
))?;
|
))?;
|
||||||
|
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let results = edge_shard.query(QueryRequest {
|
let results = edge_shard.query(QueryRequest {
|
||||||
prefetches: vec![],
|
prefetches: vec![],
|
||||||
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery {
|
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery {
|
||||||
@@ -82,6 +81,8 @@ let results = edge_shard.query(QueryRequest {
|
|||||||
with_payload: WithPayloadInterface::Bool(true),
|
with_payload: WithPayloadInterface::Bool(true),
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let filter = Filter {
|
let filter = Filter {
|
||||||
should: None,
|
should: None,
|
||||||
min_should: None,
|
min_should: None,
|
||||||
@@ -109,6 +110,8 @@ let results = edge_shard.query(QueryRequest {
|
|||||||
with_payload: WithPayloadInterface::Bool(true),
|
with_payload: WithPayloadInterface::Bool(true),
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let facet_response = edge_shard.facet(FacetRequest {
|
let facet_response = edge_shard.facet(FacetRequest {
|
||||||
key: "color".try_into().unwrap(),
|
key: "color".try_into().unwrap(),
|
||||||
limit: 10,
|
limit: 10,
|
||||||
@@ -118,6 +121,8 @@ let facet_response = edge_shard.facet(FacetRequest {
|
|||||||
|
|
||||||
edge_shard.optimize()?;
|
edge_shard.optimize()?;
|
||||||
|
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let config = EdgeConfigBuilder::new()
|
let config = EdgeConfigBuilder::new()
|
||||||
.on_disk_payload(true)
|
.on_disk_payload(true)
|
||||||
.vector(
|
.vector(
|
||||||
@@ -134,6 +139,8 @@ let config = EdgeConfigBuilder::new()
|
|||||||
})
|
})
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
edge_shard.update(UpdateOperation::FieldIndexOperation(
|
edge_shard.update(UpdateOperation::FieldIndexOperation(
|
||||||
FieldIndexOperations::CreateIndex(CreateIndex {
|
FieldIndexOperations::CreateIndex(CreateIndex {
|
||||||
field_name: "color".try_into().unwrap(),
|
field_name: "color".try_into().unwrap(),
|
||||||
@@ -145,8 +152,14 @@ edge_shard.update(UpdateOperation::FieldIndexOperation(
|
|||||||
|
|
||||||
drop(edge_shard);
|
drop(edge_shard);
|
||||||
|
|
||||||
|
use std::path::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let edge_shard = EdgeShard::load(Path::new(SHARD_DIRECTORY), None)?;
|
let edge_shard = EdgeShard::load(Path::new(SHARD_DIRECTORY), None)?;
|
||||||
|
|
||||||
|
use std::path::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let config = EdgeConfigBuilder::new()
|
let config = EdgeConfigBuilder::new()
|
||||||
.wal_options(WalOptions {
|
.wal_options(WalOptions {
|
||||||
segment_capacity: 4 * 1024 * 1024,
|
segment_capacity: 4 * 1024 * 1024,
|
||||||
|
|||||||
+3
@@ -1,4 +1,7 @@
|
|||||||
```rust
|
```rust
|
||||||
|
use serde_json::json;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let points: Vec<PointStructPersisted> = vec![
|
let points: Vec<PointStructPersisted> = vec![
|
||||||
PointStruct::new(
|
PointStruct::new(
|
||||||
1u64,
|
1u64,
|
||||||
|
|||||||
+3
@@ -1,4 +1,7 @@
|
|||||||
```rust
|
```rust
|
||||||
|
use std::path::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let config = EdgeConfigBuilder::new()
|
let config = EdgeConfigBuilder::new()
|
||||||
.wal_options(WalOptions {
|
.wal_options(WalOptions {
|
||||||
segment_capacity: 4 * 1024 * 1024,
|
segment_capacity: 4 * 1024 * 1024,
|
||||||
|
|||||||
@@ -1,26 +1,14 @@
|
|||||||
use std::path::Path;
|
// @block-start create-storage-directory
|
||||||
|
|
||||||
use qdrant_edge::EdgeShard;
|
|
||||||
use qdrant_edge::{
|
|
||||||
Condition, CreateIndex, CreateVectorName, Distance, EdgeConfigBuilder,
|
|
||||||
EdgeOptimizersConfig, EdgeVectorParamsBuilder, FacetRequest, FieldCondition,
|
|
||||||
FieldIndexOperations, Filter, Match, MatchValue, Modifier, NamedQuery,
|
|
||||||
PayloadFieldSchema, PayloadSchemaType, PointId, PointInsertOperations,
|
|
||||||
PointOperations, PointStruct, PointStructPersisted, QueryEnum, QueryRequest,
|
|
||||||
ScoringQuery, SparseVectorConfig, UpdateOperation, ValueVariants,
|
|
||||||
VectorNameConfig, VectorNameOperations, Vectors, WalOptions,
|
|
||||||
WithPayloadInterface, WithVector,
|
|
||||||
};
|
|
||||||
use serde_json::json;
|
|
||||||
|
|
||||||
pub async fn main() -> anyhow::Result<()> {
|
pub async fn main() -> anyhow::Result<()> {
|
||||||
// @block-start create-storage-directory
|
|
||||||
const SHARD_DIRECTORY: &str = "./qdrant-edge-directory";
|
const SHARD_DIRECTORY: &str = "./qdrant-edge-directory";
|
||||||
|
|
||||||
fs_err::create_dir_all(SHARD_DIRECTORY)?;
|
fs_err::create_dir_all(SHARD_DIRECTORY)?;
|
||||||
// @block-end create-storage-directory
|
// @block-end create-storage-directory
|
||||||
|
|
||||||
// @block-start configure-edge-shard
|
// @block-start configure-edge-shard
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
const VECTOR_NAME: &str = "my-vector";
|
const VECTOR_NAME: &str = "my-vector";
|
||||||
const VECTOR_DIMENSION: usize = 4;
|
const VECTOR_DIMENSION: usize = 4;
|
||||||
|
|
||||||
@@ -36,6 +24,9 @@ pub async fn main() -> anyhow::Result<()> {
|
|||||||
// @block-end configure-edge-shard
|
// @block-end configure-edge-shard
|
||||||
|
|
||||||
// @block-start initialize-edge-shard
|
// @block-start initialize-edge-shard
|
||||||
|
use std::path::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let edge_shard = EdgeShard::new(
|
let edge_shard = EdgeShard::new(
|
||||||
Path::new(SHARD_DIRECTORY),
|
Path::new(SHARD_DIRECTORY),
|
||||||
config,
|
config,
|
||||||
@@ -43,6 +34,9 @@ pub async fn main() -> anyhow::Result<()> {
|
|||||||
// @block-end initialize-edge-shard
|
// @block-end initialize-edge-shard
|
||||||
|
|
||||||
// @block-start upsert-points
|
// @block-start upsert-points
|
||||||
|
use serde_json::json;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let points: Vec<PointStructPersisted> = vec![
|
let points: Vec<PointStructPersisted> = vec![
|
||||||
PointStruct::new(
|
PointStruct::new(
|
||||||
1u64,
|
1u64,
|
||||||
@@ -60,6 +54,8 @@ pub async fn main() -> anyhow::Result<()> {
|
|||||||
// @block-end upsert-points
|
// @block-end upsert-points
|
||||||
|
|
||||||
// @block-start retrieve-point
|
// @block-start retrieve-point
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let retrieved = edge_shard.retrieve(
|
let retrieved = edge_shard.retrieve(
|
||||||
&[PointId::NumId(1)],
|
&[PointId::NumId(1)],
|
||||||
Some(WithPayloadInterface::Bool(true)),
|
Some(WithPayloadInterface::Bool(true)),
|
||||||
@@ -68,6 +64,8 @@ pub async fn main() -> anyhow::Result<()> {
|
|||||||
// @block-end retrieve-point
|
// @block-end retrieve-point
|
||||||
|
|
||||||
// @block-start modify-vector-schema
|
// @block-start modify-vector-schema
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
edge_shard.update(UpdateOperation::VectorNameOperation(
|
edge_shard.update(UpdateOperation::VectorNameOperation(
|
||||||
VectorNameOperations::CreateVectorName(CreateVectorName {
|
VectorNameOperations::CreateVectorName(CreateVectorName {
|
||||||
vector_name: "text".to_string(),
|
vector_name: "text".to_string(),
|
||||||
@@ -80,6 +78,8 @@ pub async fn main() -> anyhow::Result<()> {
|
|||||||
// @block-end modify-vector-schema
|
// @block-end modify-vector-schema
|
||||||
|
|
||||||
// @block-start query-points
|
// @block-start query-points
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let results = edge_shard.query(QueryRequest {
|
let results = edge_shard.query(QueryRequest {
|
||||||
prefetches: vec![],
|
prefetches: vec![],
|
||||||
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery {
|
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery {
|
||||||
@@ -97,6 +97,8 @@ pub async fn main() -> anyhow::Result<()> {
|
|||||||
// @block-end query-points
|
// @block-end query-points
|
||||||
|
|
||||||
// @block-start filter
|
// @block-start filter
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let filter = Filter {
|
let filter = Filter {
|
||||||
should: None,
|
should: None,
|
||||||
min_should: None,
|
min_should: None,
|
||||||
@@ -126,6 +128,8 @@ pub async fn main() -> anyhow::Result<()> {
|
|||||||
// @block-end filter
|
// @block-end filter
|
||||||
|
|
||||||
// @block-start facet
|
// @block-start facet
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let facet_response = edge_shard.facet(FacetRequest {
|
let facet_response = edge_shard.facet(FacetRequest {
|
||||||
key: "color".try_into().unwrap(),
|
key: "color".try_into().unwrap(),
|
||||||
limit: 10,
|
limit: 10,
|
||||||
@@ -139,6 +143,8 @@ pub async fn main() -> anyhow::Result<()> {
|
|||||||
// @block-end optimize
|
// @block-end optimize
|
||||||
|
|
||||||
// @block-start configure-optimizer
|
// @block-start configure-optimizer
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let config = EdgeConfigBuilder::new()
|
let config = EdgeConfigBuilder::new()
|
||||||
.on_disk_payload(true)
|
.on_disk_payload(true)
|
||||||
.vector(
|
.vector(
|
||||||
@@ -157,6 +163,8 @@ pub async fn main() -> anyhow::Result<()> {
|
|||||||
// @block-end configure-optimizer
|
// @block-end configure-optimizer
|
||||||
|
|
||||||
// @block-start create-payload-index
|
// @block-start create-payload-index
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
edge_shard.update(UpdateOperation::FieldIndexOperation(
|
edge_shard.update(UpdateOperation::FieldIndexOperation(
|
||||||
FieldIndexOperations::CreateIndex(CreateIndex {
|
FieldIndexOperations::CreateIndex(CreateIndex {
|
||||||
field_name: "color".try_into().unwrap(),
|
field_name: "color".try_into().unwrap(),
|
||||||
@@ -172,10 +180,16 @@ pub async fn main() -> anyhow::Result<()> {
|
|||||||
// @block-end close-edge-shard
|
// @block-end close-edge-shard
|
||||||
|
|
||||||
// @block-start load-edge-shard
|
// @block-start load-edge-shard
|
||||||
|
use std::path::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let edge_shard = EdgeShard::load(Path::new(SHARD_DIRECTORY), None)?;
|
let edge_shard = EdgeShard::load(Path::new(SHARD_DIRECTORY), None)?;
|
||||||
// @block-end load-edge-shard
|
// @block-end load-edge-shard
|
||||||
|
|
||||||
// @block-start wal-options
|
// @block-start wal-options
|
||||||
|
use std::path::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let config = EdgeConfigBuilder::new()
|
let config = EdgeConfigBuilder::new()
|
||||||
.wal_options(WalOptions {
|
.wal_options(WalOptions {
|
||||||
segment_capacity: 4 * 1024 * 1024,
|
segment_capacity: 4 * 1024 * 1024,
|
||||||
|
|||||||
+3
@@ -1,4 +1,7 @@
|
|||||||
```rust
|
```rust
|
||||||
|
use ordered_float::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let filter = Filter::new_must(Condition::Field(FieldCondition::new_range(
|
let filter = Filter::new_must(Condition::Field(FieldCondition::new_range(
|
||||||
SYNC_TIMESTAMP_KEY.parse::<JsonPath>().unwrap(),
|
SYNC_TIMESTAMP_KEY.parse::<JsonPath>().unwrap(),
|
||||||
Range {
|
Range {
|
||||||
|
|||||||
+7
-2
@@ -1,4 +1,9 @@
|
|||||||
```rust
|
```rust
|
||||||
|
use std::collections::*;
|
||||||
|
use std::time::*;
|
||||||
|
use serde_json::json;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
const SYNC_TIMESTAMP_KEY: &str = "timestamp";
|
const SYNC_TIMESTAMP_KEY: &str = "timestamp";
|
||||||
let id = 2u64;
|
let id = 2u64;
|
||||||
let vector = vec![0.4f32, 0.3, 0.2, 0.1];
|
let vector = vec![0.4f32, 0.3, 0.2, 0.1];
|
||||||
@@ -12,7 +17,7 @@ let payload = json!({
|
|||||||
});
|
});
|
||||||
|
|
||||||
let edge_points: Vec<PointStructPersisted> = vec![
|
let edge_points: Vec<PointStructPersisted> = vec![
|
||||||
EdgePoint::new(
|
qdrant_edge::PointStruct::new(
|
||||||
PointId::NumId(id),
|
PointId::NumId(id),
|
||||||
Vectors::new_named([(VECTOR_NAME, vector.clone())]),
|
Vectors::new_named([(VECTOR_NAME, vector.clone())]),
|
||||||
payload.clone(),
|
payload.clone(),
|
||||||
@@ -25,7 +30,7 @@ mutable_shard.update(UpdateOperation::PointOperation(
|
|||||||
),
|
),
|
||||||
))?;
|
))?;
|
||||||
|
|
||||||
let rest_point = PointStruct::new(
|
let rest_point = ::qdrant_client::qdrant::PointStruct::new(
|
||||||
id,
|
id,
|
||||||
HashMap::from([(VECTOR_NAME.to_string(), vector)]),
|
HashMap::from([(VECTOR_NAME.to_string(), vector)]),
|
||||||
payload.as_object().cloned().unwrap_or_default(),
|
payload.as_object().cloned().unwrap_or_default(),
|
||||||
|
|||||||
+3
@@ -1,4 +1,7 @@
|
|||||||
```rust
|
```rust
|
||||||
|
use std::path::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
const COLLECTION_NAME: &str = "edge-collection";
|
const COLLECTION_NAME: &str = "edge-collection";
|
||||||
let snapshot_url = format!(
|
let snapshot_url = format!(
|
||||||
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot"
|
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot"
|
||||||
|
|||||||
+3
@@ -1,4 +1,7 @@
|
|||||||
```rust
|
```rust
|
||||||
|
use std::path::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
const MUTABLE_SHARD_DIR: &str = "./qdrant-edge-directory/mutable";
|
const MUTABLE_SHARD_DIR: &str = "./qdrant-edge-directory/mutable";
|
||||||
const VECTOR_DIMENSION: usize = 4;
|
const VECTOR_DIMENSION: usize = 4;
|
||||||
const VECTOR_NAME: &str = "my-vector";
|
const VECTOR_NAME: &str = "my-vector";
|
||||||
|
|||||||
+6
-2
@@ -1,4 +1,8 @@
|
|||||||
```rust
|
```rust
|
||||||
|
use std::cmp::*;
|
||||||
|
use std::collections::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let query = QueryRequest {
|
let query = QueryRequest {
|
||||||
prefetches: vec![],
|
prefetches: vec![],
|
||||||
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery {
|
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery {
|
||||||
@@ -20,10 +24,10 @@ all_results.extend(immutable_shard.query(query)?);
|
|||||||
all_results.sort_by(|a, b| {
|
all_results.sort_by(|a, b| {
|
||||||
b.score
|
b.score
|
||||||
.partial_cmp(&a.score)
|
.partial_cmp(&a.score)
|
||||||
.unwrap_or(std::cmp::Ordering::Equal)
|
.unwrap_or(Ordering::Equal)
|
||||||
});
|
});
|
||||||
|
|
||||||
let mut seen_ids = std::collections::HashSet::new();
|
let mut seen_ids = HashSet::new();
|
||||||
let results: Vec<_> = all_results
|
let results: Vec<_> = all_results
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.filter(|p| seen_ids.insert(p.id.clone()))
|
.filter(|p| seen_ids.insert(p.id.clone()))
|
||||||
|
|||||||
+25
-20
@@ -1,20 +1,6 @@
|
|||||||
```rust
|
```rust
|
||||||
use std::collections::HashMap;
|
use std::path::*;
|
||||||
use std::path::Path;
|
use qdrant_edge::*;
|
||||||
use std::time::{SystemTime, UNIX_EPOCH};
|
|
||||||
|
|
||||||
use ordered_float::OrderedFloat;
|
|
||||||
use qdrant_client::qdrant::PointStruct;
|
|
||||||
use qdrant_edge::EdgeShard;
|
|
||||||
use qdrant_edge::internal::SnapshotManifest;
|
|
||||||
use qdrant_edge::{
|
|
||||||
Condition, Distance, EdgeConfigBuilder, EdgeVectorParamsBuilder,
|
|
||||||
FieldCondition, Filter, JsonPath, NamedQuery, PointId, PointInsertOperations,
|
|
||||||
PointOperations, PointStruct as EdgePoint, PointStructPersisted, QueryEnum,
|
|
||||||
QueryRequest, Range, ScoringQuery, UpdateOperation, Vectors,
|
|
||||||
WithPayloadInterface, WithVector,
|
|
||||||
};
|
|
||||||
use serde_json::json;
|
|
||||||
|
|
||||||
const MUTABLE_SHARD_DIR: &str = "./qdrant-edge-directory/mutable";
|
const MUTABLE_SHARD_DIR: &str = "./qdrant-edge-directory/mutable";
|
||||||
const VECTOR_DIMENSION: usize = 4;
|
const VECTOR_DIMENSION: usize = 4;
|
||||||
@@ -36,6 +22,9 @@ let mutable_shard = EdgeShard::new(
|
|||||||
config,
|
config,
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
|
use std::path::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
const COLLECTION_NAME: &str = "edge-collection";
|
const COLLECTION_NAME: &str = "edge-collection";
|
||||||
let snapshot_url = format!(
|
let snapshot_url = format!(
|
||||||
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot"
|
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot"
|
||||||
@@ -68,6 +57,11 @@ EdgeShard::unpack_snapshot(&snapshot_path, data_dir)?;
|
|||||||
|
|
||||||
let immutable_shard = EdgeShard::load(data_dir, None)?;
|
let immutable_shard = EdgeShard::load(data_dir, None)?;
|
||||||
|
|
||||||
|
use std::collections::*;
|
||||||
|
use std::time::*;
|
||||||
|
use serde_json::json;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
const SYNC_TIMESTAMP_KEY: &str = "timestamp";
|
const SYNC_TIMESTAMP_KEY: &str = "timestamp";
|
||||||
let id = 2u64;
|
let id = 2u64;
|
||||||
let vector = vec![0.4f32, 0.3, 0.2, 0.1];
|
let vector = vec![0.4f32, 0.3, 0.2, 0.1];
|
||||||
@@ -81,7 +75,7 @@ let payload = json!({
|
|||||||
});
|
});
|
||||||
|
|
||||||
let edge_points: Vec<PointStructPersisted> = vec![
|
let edge_points: Vec<PointStructPersisted> = vec![
|
||||||
EdgePoint::new(
|
qdrant_edge::PointStruct::new(
|
||||||
PointId::NumId(id),
|
PointId::NumId(id),
|
||||||
Vectors::new_named([(VECTOR_NAME, vector.clone())]),
|
Vectors::new_named([(VECTOR_NAME, vector.clone())]),
|
||||||
payload.clone(),
|
payload.clone(),
|
||||||
@@ -94,13 +88,17 @@ mutable_shard.update(UpdateOperation::PointOperation(
|
|||||||
),
|
),
|
||||||
))?;
|
))?;
|
||||||
|
|
||||||
let rest_point = PointStruct::new(
|
let rest_point = ::qdrant_client::qdrant::PointStruct::new(
|
||||||
id,
|
id,
|
||||||
HashMap::from([(VECTOR_NAME.to_string(), vector)]),
|
HashMap::from([(VECTOR_NAME.to_string(), vector)]),
|
||||||
payload.as_object().cloned().unwrap_or_default(),
|
payload.as_object().cloned().unwrap_or_default(),
|
||||||
);
|
);
|
||||||
upload_queue.push_back(rest_point);
|
upload_queue.push_back(rest_point);
|
||||||
|
|
||||||
|
use std::time::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
use qdrant_edge::internal::*;
|
||||||
|
|
||||||
let sync_timestamp = SystemTime::now()
|
let sync_timestamp = SystemTime::now()
|
||||||
.duration_since(UNIX_EPOCH)
|
.duration_since(UNIX_EPOCH)
|
||||||
.unwrap()
|
.unwrap()
|
||||||
@@ -141,6 +139,9 @@ let immutable_shard = EdgeShard::recover_partial_snapshot(
|
|||||||
&snapshot_manifest,
|
&snapshot_manifest,
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
|
use ordered_float::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let filter = Filter::new_must(Condition::Field(FieldCondition::new_range(
|
let filter = Filter::new_must(Condition::Field(FieldCondition::new_range(
|
||||||
SYNC_TIMESTAMP_KEY.parse::<JsonPath>().unwrap(),
|
SYNC_TIMESTAMP_KEY.parse::<JsonPath>().unwrap(),
|
||||||
Range {
|
Range {
|
||||||
@@ -153,6 +154,10 @@ mutable_shard.update(UpdateOperation::PointOperation(
|
|||||||
PointOperations::DeletePointsByFilter(filter),
|
PointOperations::DeletePointsByFilter(filter),
|
||||||
))?;
|
))?;
|
||||||
|
|
||||||
|
use std::cmp::*;
|
||||||
|
use std::collections::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let query = QueryRequest {
|
let query = QueryRequest {
|
||||||
prefetches: vec![],
|
prefetches: vec![],
|
||||||
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery {
|
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery {
|
||||||
@@ -174,10 +179,10 @@ all_results.extend(immutable_shard.query(query)?);
|
|||||||
all_results.sort_by(|a, b| {
|
all_results.sort_by(|a, b| {
|
||||||
b.score
|
b.score
|
||||||
.partial_cmp(&a.score)
|
.partial_cmp(&a.score)
|
||||||
.unwrap_or(std::cmp::Ordering::Equal)
|
.unwrap_or(Ordering::Equal)
|
||||||
});
|
});
|
||||||
|
|
||||||
let mut seen_ids = std::collections::HashSet::new();
|
let mut seen_ids = HashSet::new();
|
||||||
let results: Vec<_> = all_results
|
let results: Vec<_> = all_results
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.filter(|p| seen_ids.insert(p.id.clone()))
|
.filter(|p| seen_ids.insert(p.id.clone()))
|
||||||
|
|||||||
+4
@@ -1,4 +1,8 @@
|
|||||||
```rust
|
```rust
|
||||||
|
use std::time::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
use qdrant_edge::internal::*;
|
||||||
|
|
||||||
let sync_timestamp = SystemTime::now()
|
let sync_timestamp = SystemTime::now()
|
||||||
.duration_since(UNIX_EPOCH)
|
.duration_since(UNIX_EPOCH)
|
||||||
.unwrap()
|
.unwrap()
|
||||||
|
|||||||
+28
-23
@@ -1,29 +1,15 @@
|
|||||||
use std::collections::HashMap;
|
// @block-start initialize-mutable-shard
|
||||||
use std::path::Path;
|
|
||||||
use std::time::{SystemTime, UNIX_EPOCH};
|
|
||||||
|
|
||||||
use ordered_float::OrderedFloat;
|
|
||||||
use qdrant_client::qdrant::PointStruct;
|
|
||||||
use qdrant_edge::EdgeShard;
|
|
||||||
use qdrant_edge::internal::SnapshotManifest;
|
|
||||||
use qdrant_edge::{
|
|
||||||
Condition, Distance, EdgeConfigBuilder, EdgeVectorParamsBuilder,
|
|
||||||
FieldCondition, Filter, JsonPath, NamedQuery, PointId, PointInsertOperations,
|
|
||||||
PointOperations, PointStruct as EdgePoint, PointStructPersisted, QueryEnum,
|
|
||||||
QueryRequest, Range, ScoringQuery, UpdateOperation, Vectors,
|
|
||||||
WithPayloadInterface, WithVector,
|
|
||||||
};
|
|
||||||
use serde_json::json;
|
|
||||||
|
|
||||||
pub async fn main() -> anyhow::Result<()> {
|
pub async fn main() -> anyhow::Result<()> {
|
||||||
// @hide-start
|
// @hide-start
|
||||||
const QDRANT_URL: &str = "";
|
const QDRANT_URL: &str = "";
|
||||||
const QDRANT_API_KEY: &str = "";
|
const QDRANT_API_KEY: &str = "";
|
||||||
let mut upload_queue: std::collections::VecDeque<PointStruct> =
|
let mut upload_queue: std::collections::VecDeque<::qdrant_client::qdrant::PointStruct> =
|
||||||
std::collections::VecDeque::new();
|
std::collections::VecDeque::new();
|
||||||
// @hide-end
|
// @hide-end
|
||||||
|
|
||||||
// @block-start initialize-mutable-shard
|
use std::path::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
const MUTABLE_SHARD_DIR: &str = "./qdrant-edge-directory/mutable";
|
const MUTABLE_SHARD_DIR: &str = "./qdrant-edge-directory/mutable";
|
||||||
const VECTOR_DIMENSION: usize = 4;
|
const VECTOR_DIMENSION: usize = 4;
|
||||||
const VECTOR_NAME: &str = "my-vector";
|
const VECTOR_NAME: &str = "my-vector";
|
||||||
@@ -46,6 +32,9 @@ pub async fn main() -> anyhow::Result<()> {
|
|||||||
// @block-end initialize-mutable-shard
|
// @block-end initialize-mutable-shard
|
||||||
|
|
||||||
// @block-start initialize-immutable-shard
|
// @block-start initialize-immutable-shard
|
||||||
|
use std::path::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
const COLLECTION_NAME: &str = "edge-collection";
|
const COLLECTION_NAME: &str = "edge-collection";
|
||||||
let snapshot_url = format!(
|
let snapshot_url = format!(
|
||||||
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot"
|
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot"
|
||||||
@@ -80,6 +69,11 @@ pub async fn main() -> anyhow::Result<()> {
|
|||||||
// @block-end initialize-immutable-shard
|
// @block-end initialize-immutable-shard
|
||||||
|
|
||||||
// @block-start dual-write
|
// @block-start dual-write
|
||||||
|
use std::collections::*;
|
||||||
|
use std::time::*;
|
||||||
|
use serde_json::json;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
const SYNC_TIMESTAMP_KEY: &str = "timestamp";
|
const SYNC_TIMESTAMP_KEY: &str = "timestamp";
|
||||||
let id = 2u64;
|
let id = 2u64;
|
||||||
let vector = vec![0.4f32, 0.3, 0.2, 0.1];
|
let vector = vec![0.4f32, 0.3, 0.2, 0.1];
|
||||||
@@ -93,7 +87,7 @@ pub async fn main() -> anyhow::Result<()> {
|
|||||||
});
|
});
|
||||||
|
|
||||||
let edge_points: Vec<PointStructPersisted> = vec![
|
let edge_points: Vec<PointStructPersisted> = vec![
|
||||||
EdgePoint::new(
|
qdrant_edge::PointStruct::new(
|
||||||
PointId::NumId(id),
|
PointId::NumId(id),
|
||||||
Vectors::new_named([(VECTOR_NAME, vector.clone())]),
|
Vectors::new_named([(VECTOR_NAME, vector.clone())]),
|
||||||
payload.clone(),
|
payload.clone(),
|
||||||
@@ -106,7 +100,7 @@ pub async fn main() -> anyhow::Result<()> {
|
|||||||
),
|
),
|
||||||
))?;
|
))?;
|
||||||
|
|
||||||
let rest_point = PointStruct::new(
|
let rest_point = ::qdrant_client::qdrant::PointStruct::new(
|
||||||
id,
|
id,
|
||||||
HashMap::from([(VECTOR_NAME.to_string(), vector)]),
|
HashMap::from([(VECTOR_NAME.to_string(), vector)]),
|
||||||
payload.as_object().cloned().unwrap_or_default(),
|
payload.as_object().cloned().unwrap_or_default(),
|
||||||
@@ -115,6 +109,10 @@ pub async fn main() -> anyhow::Result<()> {
|
|||||||
// @block-end dual-write
|
// @block-end dual-write
|
||||||
|
|
||||||
// @block-start update-immutable-shard
|
// @block-start update-immutable-shard
|
||||||
|
use std::time::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
use qdrant_edge::internal::*;
|
||||||
|
|
||||||
let sync_timestamp = SystemTime::now()
|
let sync_timestamp = SystemTime::now()
|
||||||
.duration_since(UNIX_EPOCH)
|
.duration_since(UNIX_EPOCH)
|
||||||
.unwrap()
|
.unwrap()
|
||||||
@@ -157,6 +155,9 @@ pub async fn main() -> anyhow::Result<()> {
|
|||||||
// @block-end update-immutable-shard
|
// @block-end update-immutable-shard
|
||||||
|
|
||||||
// @block-start delete-synced-points
|
// @block-start delete-synced-points
|
||||||
|
use ordered_float::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let filter = Filter::new_must(Condition::Field(FieldCondition::new_range(
|
let filter = Filter::new_must(Condition::Field(FieldCondition::new_range(
|
||||||
SYNC_TIMESTAMP_KEY.parse::<JsonPath>().unwrap(),
|
SYNC_TIMESTAMP_KEY.parse::<JsonPath>().unwrap(),
|
||||||
Range {
|
Range {
|
||||||
@@ -171,6 +172,10 @@ pub async fn main() -> anyhow::Result<()> {
|
|||||||
// @block-end delete-synced-points
|
// @block-end delete-synced-points
|
||||||
|
|
||||||
// @block-start query-both-shards
|
// @block-start query-both-shards
|
||||||
|
use std::cmp::*;
|
||||||
|
use std::collections::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let query = QueryRequest {
|
let query = QueryRequest {
|
||||||
prefetches: vec![],
|
prefetches: vec![],
|
||||||
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery {
|
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery {
|
||||||
@@ -192,10 +197,10 @@ pub async fn main() -> anyhow::Result<()> {
|
|||||||
all_results.sort_by(|a, b| {
|
all_results.sort_by(|a, b| {
|
||||||
b.score
|
b.score
|
||||||
.partial_cmp(&a.score)
|
.partial_cmp(&a.score)
|
||||||
.unwrap_or(std::cmp::Ordering::Equal)
|
.unwrap_or(Ordering::Equal)
|
||||||
});
|
});
|
||||||
|
|
||||||
let mut seen_ids = std::collections::HashSet::new();
|
let mut seen_ids = HashSet::new();
|
||||||
let results: Vec<_> = all_results
|
let results: Vec<_> = all_results
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.filter(|p| seen_ids.insert(p.id.clone()))
|
.filter(|p| seen_ids.insert(p.id.clone()))
|
||||||
|
|||||||
+3
-2
@@ -1,6 +1,7 @@
|
|||||||
```rust
|
```rust
|
||||||
|
use std::collections::*;
|
||||||
|
|
||||||
// This is an in-memory queue.
|
// This is an in-memory queue.
|
||||||
// For production use cases consider persisting changes.
|
// For production use cases consider persisting changes.
|
||||||
let mut upload_queue: std::collections::VecDeque<PointStruct> =
|
let mut upload_queue: VecDeque<::qdrant_client::qdrant::PointStruct> = VecDeque::new();
|
||||||
std::collections::VecDeque::new();
|
|
||||||
```
|
```
|
||||||
|
|||||||
+3
@@ -1,4 +1,7 @@
|
|||||||
```rust
|
```rust
|
||||||
|
use std::path::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
const VECTOR_DIMENSION: usize = 4;
|
const VECTOR_DIMENSION: usize = 4;
|
||||||
const VECTOR_NAME: &str = "my-vector";
|
const VECTOR_NAME: &str = "my-vector";
|
||||||
|
|
||||||
|
|||||||
+4
@@ -1,4 +1,8 @@
|
|||||||
```rust
|
```rust
|
||||||
|
use ::qdrant_client::*;
|
||||||
|
use ::qdrant_client::qdrant::*;
|
||||||
|
use ::qdrant_client::qdrant::Distance;
|
||||||
|
|
||||||
let server_client = Qdrant::from_url(QDRANT_URL)
|
let server_client = Qdrant::from_url(QDRANT_URL)
|
||||||
.api_key(QDRANT_API_KEY)
|
.api_key(QDRANT_API_KEY)
|
||||||
.build()?;
|
.build()?;
|
||||||
|
|||||||
+3
-1
@@ -1,6 +1,8 @@
|
|||||||
```rust
|
```rust
|
||||||
|
use ::qdrant_client::qdrant::*;
|
||||||
|
|
||||||
const BATCH_SIZE: usize = 10;
|
const BATCH_SIZE: usize = 10;
|
||||||
let points_to_upload: Vec<PointStruct> = upload_queue
|
let points_to_upload: Vec<::qdrant_client::qdrant::PointStruct> = upload_queue
|
||||||
.drain(..BATCH_SIZE.min(upload_queue.len()))
|
.drain(..BATCH_SIZE.min(upload_queue.len()))
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
|
|||||||
+4
-4
@@ -1,14 +1,14 @@
|
|||||||
```python
|
```python
|
||||||
COLLECTION_NAME="edge-collection"
|
|
||||||
|
|
||||||
snapshot_url = f"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot"
|
|
||||||
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from qdrant_edge import EdgeShard
|
from qdrant_edge import EdgeShard
|
||||||
import requests
|
import requests
|
||||||
import shutil
|
import shutil
|
||||||
import tempfile
|
import tempfile
|
||||||
|
|
||||||
|
COLLECTION_NAME="edge-collection"
|
||||||
|
|
||||||
|
snapshot_url = f"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot"
|
||||||
|
|
||||||
SHARD_DIRECTORY = "./qdrant-edge-directory"
|
SHARD_DIRECTORY = "./qdrant-edge-directory"
|
||||||
data_dir = Path(SHARD_DIRECTORY)
|
data_dir = Path(SHARD_DIRECTORY)
|
||||||
|
|
||||||
|
|||||||
+4
@@ -5,6 +5,10 @@ import requests
|
|||||||
import shutil
|
import shutil
|
||||||
import tempfile
|
import tempfile
|
||||||
|
|
||||||
|
COLLECTION_NAME="edge-collection"
|
||||||
|
|
||||||
|
snapshot_url = f"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot"
|
||||||
|
|
||||||
SHARD_DIRECTORY = "./qdrant-edge-directory"
|
SHARD_DIRECTORY = "./qdrant-edge-directory"
|
||||||
data_dir = Path(SHARD_DIRECTORY)
|
data_dir = Path(SHARD_DIRECTORY)
|
||||||
|
|
||||||
|
|||||||
+9
@@ -1,5 +1,14 @@
|
|||||||
```rust
|
```rust
|
||||||
|
const COLLECTION_NAME: &str = "edge-collection";
|
||||||
const SHARD_DIRECTORY: &str = "./qdrant-edge-directory";
|
const SHARD_DIRECTORY: &str = "./qdrant-edge-directory";
|
||||||
|
|
||||||
|
use std::path::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
|
let snapshot_url = format!(
|
||||||
|
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot"
|
||||||
|
);
|
||||||
|
|
||||||
let data_dir = Path::new(SHARD_DIRECTORY);
|
let data_dir = Path::new(SHARD_DIRECTORY);
|
||||||
|
|
||||||
let restore_dir = tempfile::Builder::new()
|
let restore_dir = tempfile::Builder::new()
|
||||||
|
|||||||
+29
-23
@@ -1,28 +1,14 @@
|
|||||||
```rust
|
```rust
|
||||||
use std::collections::HashMap;
|
|
||||||
use std::path::Path;
|
|
||||||
|
|
||||||
use qdrant_client::qdrant::{
|
|
||||||
CreateCollectionBuilder, Distance, PointStruct, UpsertPointsBuilder,
|
|
||||||
VectorParamsBuilder,
|
|
||||||
};
|
|
||||||
use qdrant_client::Qdrant;
|
|
||||||
use qdrant_edge::EdgeShard;
|
|
||||||
use qdrant_edge::internal::SnapshotManifest;
|
|
||||||
use qdrant_edge::{
|
|
||||||
EdgeConfigBuilder, EdgeVectorParamsBuilder, PointId, PointInsertOperations,
|
|
||||||
PointOperations, PointStruct as EdgePoint, PointStructPersisted,
|
|
||||||
UpdateOperation, Vectors,
|
|
||||||
};
|
|
||||||
use serde_json::json;
|
|
||||||
|
|
||||||
const COLLECTION_NAME: &str = "edge-collection";
|
const COLLECTION_NAME: &str = "edge-collection";
|
||||||
|
const SHARD_DIRECTORY: &str = "./qdrant-edge-directory";
|
||||||
|
|
||||||
|
use std::path::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let snapshot_url = format!(
|
let snapshot_url = format!(
|
||||||
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot"
|
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot"
|
||||||
);
|
);
|
||||||
|
|
||||||
const SHARD_DIRECTORY: &str = "./qdrant-edge-directory";
|
|
||||||
let data_dir = Path::new(SHARD_DIRECTORY);
|
let data_dir = Path::new(SHARD_DIRECTORY);
|
||||||
|
|
||||||
let restore_dir = tempfile::Builder::new()
|
let restore_dir = tempfile::Builder::new()
|
||||||
@@ -49,6 +35,9 @@ EdgeShard::unpack_snapshot(&snapshot_path, data_dir)?;
|
|||||||
|
|
||||||
let edge_shard = EdgeShard::load(data_dir, None)?;
|
let edge_shard = EdgeShard::load(data_dir, None)?;
|
||||||
|
|
||||||
|
use qdrant_edge::*;
|
||||||
|
use qdrant_edge::internal::*;
|
||||||
|
|
||||||
let current_manifest = edge_shard.snapshot_manifest()?;
|
let current_manifest = edge_shard.snapshot_manifest()?;
|
||||||
|
|
||||||
let update_url = format!(
|
let update_url = format!(
|
||||||
@@ -84,6 +73,9 @@ let edge_shard = EdgeShard::recover_partial_snapshot(
|
|||||||
&snapshot_manifest,
|
&snapshot_manifest,
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
|
use std::path::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
const VECTOR_DIMENSION: usize = 4;
|
const VECTOR_DIMENSION: usize = 4;
|
||||||
const VECTOR_NAME: &str = "my-vector";
|
const VECTOR_NAME: &str = "my-vector";
|
||||||
|
|
||||||
@@ -103,6 +95,10 @@ let edge_shard = EdgeShard::new(
|
|||||||
config,
|
config,
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
|
use ::qdrant_client::*;
|
||||||
|
use ::qdrant_client::qdrant::*;
|
||||||
|
use ::qdrant_client::qdrant::Distance;
|
||||||
|
|
||||||
let server_client = Qdrant::from_url(QDRANT_URL)
|
let server_client = Qdrant::from_url(QDRANT_URL)
|
||||||
.api_key(QDRANT_API_KEY)
|
.api_key(QDRANT_API_KEY)
|
||||||
.build()?;
|
.build()?;
|
||||||
@@ -120,17 +116,25 @@ if !server_client.collection_exists(COLLECTION_NAME).await? {
|
|||||||
.await?;
|
.await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
use std::collections::*;
|
||||||
|
|
||||||
// This is an in-memory queue.
|
// This is an in-memory queue.
|
||||||
// For production use cases consider persisting changes.
|
// For production use cases consider persisting changes.
|
||||||
let mut upload_queue: std::collections::VecDeque<PointStruct> =
|
let mut upload_queue: VecDeque<::qdrant_client::qdrant::PointStruct> = VecDeque::new();
|
||||||
std::collections::VecDeque::new();
|
|
||||||
|
use std::collections::*;
|
||||||
|
use serde_json::json;
|
||||||
|
use qdrant_edge::{
|
||||||
|
PointInsertOperations, PointOperations,
|
||||||
|
PointStructPersisted, PointId, UpdateOperation, Vectors,
|
||||||
|
};
|
||||||
|
|
||||||
let id = 1u64;
|
let id = 1u64;
|
||||||
let vector = vec![0.1f32, 0.2, 0.3, 0.4];
|
let vector = vec![0.1f32, 0.2, 0.3, 0.4];
|
||||||
let payload = json!({"color": "red"});
|
let payload = json!({"color": "red"});
|
||||||
|
|
||||||
let edge_points: Vec<PointStructPersisted> = vec![
|
let edge_points: Vec<PointStructPersisted> = vec![
|
||||||
EdgePoint::new(
|
qdrant_edge::PointStruct::new(
|
||||||
PointId::NumId(id),
|
PointId::NumId(id),
|
||||||
Vectors::new_named([(VECTOR_NAME, vector.clone())]),
|
Vectors::new_named([(VECTOR_NAME, vector.clone())]),
|
||||||
payload.clone(),
|
payload.clone(),
|
||||||
@@ -143,15 +147,17 @@ edge_shard.update(UpdateOperation::PointOperation(
|
|||||||
),
|
),
|
||||||
))?;
|
))?;
|
||||||
|
|
||||||
let server_point = PointStruct::new(
|
let server_point = ::qdrant_client::qdrant::PointStruct::new(
|
||||||
id,
|
id,
|
||||||
HashMap::from([(VECTOR_NAME.to_string(), vector)]),
|
HashMap::from([(VECTOR_NAME.to_string(), vector)]),
|
||||||
payload.as_object().cloned().unwrap_or_default(),
|
payload.as_object().cloned().unwrap_or_default(),
|
||||||
);
|
);
|
||||||
upload_queue.push_back(server_point);
|
upload_queue.push_back(server_point);
|
||||||
|
|
||||||
|
use ::qdrant_client::qdrant::*;
|
||||||
|
|
||||||
const BATCH_SIZE: usize = 10;
|
const BATCH_SIZE: usize = 10;
|
||||||
let points_to_upload: Vec<PointStruct> = upload_queue
|
let points_to_upload: Vec<::qdrant_client::qdrant::PointStruct> = upload_queue
|
||||||
.drain(..BATCH_SIZE.min(upload_queue.len()))
|
.drain(..BATCH_SIZE.min(upload_queue.len()))
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
|
|||||||
-5
@@ -1,5 +0,0 @@
|
|||||||
```python
|
|
||||||
COLLECTION_NAME="edge-collection"
|
|
||||||
|
|
||||||
snapshot_url = f"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot"
|
|
||||||
```
|
|
||||||
-7
@@ -1,7 +0,0 @@
|
|||||||
```rust
|
|
||||||
const COLLECTION_NAME: &str = "edge-collection";
|
|
||||||
|
|
||||||
let snapshot_url = format!(
|
|
||||||
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot"
|
|
||||||
);
|
|
||||||
```
|
|
||||||
+3
@@ -1,4 +1,7 @@
|
|||||||
```rust
|
```rust
|
||||||
|
use qdrant_edge::*;
|
||||||
|
use qdrant_edge::internal::*;
|
||||||
|
|
||||||
let current_manifest = edge_shard.snapshot_manifest()?;
|
let current_manifest = edge_shard.snapshot_manifest()?;
|
||||||
|
|
||||||
let update_url = format!(
|
let update_url = format!(
|
||||||
|
|||||||
+9
-2
@@ -1,10 +1,17 @@
|
|||||||
```rust
|
```rust
|
||||||
|
use std::collections::*;
|
||||||
|
use serde_json::json;
|
||||||
|
use qdrant_edge::{
|
||||||
|
PointInsertOperations, PointOperations,
|
||||||
|
PointStructPersisted, PointId, UpdateOperation, Vectors,
|
||||||
|
};
|
||||||
|
|
||||||
let id = 1u64;
|
let id = 1u64;
|
||||||
let vector = vec![0.1f32, 0.2, 0.3, 0.4];
|
let vector = vec![0.1f32, 0.2, 0.3, 0.4];
|
||||||
let payload = json!({"color": "red"});
|
let payload = json!({"color": "red"});
|
||||||
|
|
||||||
let edge_points: Vec<PointStructPersisted> = vec![
|
let edge_points: Vec<PointStructPersisted> = vec![
|
||||||
EdgePoint::new(
|
qdrant_edge::PointStruct::new(
|
||||||
PointId::NumId(id),
|
PointId::NumId(id),
|
||||||
Vectors::new_named([(VECTOR_NAME, vector.clone())]),
|
Vectors::new_named([(VECTOR_NAME, vector.clone())]),
|
||||||
payload.clone(),
|
payload.clone(),
|
||||||
@@ -17,7 +24,7 @@ edge_shard.update(UpdateOperation::PointOperation(
|
|||||||
),
|
),
|
||||||
))?;
|
))?;
|
||||||
|
|
||||||
let server_point = PointStruct::new(
|
let server_point = ::qdrant_client::qdrant::PointStruct::new(
|
||||||
id,
|
id,
|
||||||
HashMap::from([(VECTOR_NAME.to_string(), vector)]),
|
HashMap::from([(VECTOR_NAME.to_string(), vector)]),
|
||||||
payload.as_object().cloned().unwrap_or_default(),
|
payload.as_object().cloned().unwrap_or_default(),
|
||||||
|
|||||||
+4
-6
@@ -4,12 +4,6 @@ QDRANT_URL=""
|
|||||||
QDRANT_API_KEY=""
|
QDRANT_API_KEY=""
|
||||||
# @hide-end
|
# @hide-end
|
||||||
|
|
||||||
# @block-start snapshot-url
|
|
||||||
COLLECTION_NAME="edge-collection"
|
|
||||||
|
|
||||||
snapshot_url = f"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot"
|
|
||||||
# @block-end snapshot-url
|
|
||||||
|
|
||||||
# @block-start restore-snapshot
|
# @block-start restore-snapshot
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from qdrant_edge import EdgeShard
|
from qdrant_edge import EdgeShard
|
||||||
@@ -17,6 +11,10 @@ import requests
|
|||||||
import shutil
|
import shutil
|
||||||
import tempfile
|
import tempfile
|
||||||
|
|
||||||
|
COLLECTION_NAME="edge-collection"
|
||||||
|
|
||||||
|
snapshot_url = f"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot"
|
||||||
|
|
||||||
SHARD_DIRECTORY = "./qdrant-edge-directory"
|
SHARD_DIRECTORY = "./qdrant-edge-directory"
|
||||||
data_dir = Path(SHARD_DIRECTORY)
|
data_dir = Path(SHARD_DIRECTORY)
|
||||||
|
|
||||||
|
|||||||
+30
-26
@@ -1,36 +1,20 @@
|
|||||||
use std::collections::HashMap;
|
// @block-start restore-snapshot
|
||||||
use std::path::Path;
|
|
||||||
|
|
||||||
use qdrant_client::qdrant::{
|
|
||||||
CreateCollectionBuilder, Distance, PointStruct, UpsertPointsBuilder,
|
|
||||||
VectorParamsBuilder,
|
|
||||||
};
|
|
||||||
use qdrant_client::Qdrant;
|
|
||||||
use qdrant_edge::EdgeShard;
|
|
||||||
use qdrant_edge::internal::SnapshotManifest;
|
|
||||||
use qdrant_edge::{
|
|
||||||
EdgeConfigBuilder, EdgeVectorParamsBuilder, PointId, PointInsertOperations,
|
|
||||||
PointOperations, PointStruct as EdgePoint, PointStructPersisted,
|
|
||||||
UpdateOperation, Vectors,
|
|
||||||
};
|
|
||||||
use serde_json::json;
|
|
||||||
|
|
||||||
pub async fn main() -> anyhow::Result<()> {
|
pub async fn main() -> anyhow::Result<()> {
|
||||||
// @hide-start
|
// @hide-start
|
||||||
const QDRANT_URL: &str = "";
|
const QDRANT_URL: &str = "";
|
||||||
const QDRANT_API_KEY: &str = "";
|
const QDRANT_API_KEY: &str = "";
|
||||||
// @hide-end
|
// @hide-end
|
||||||
|
|
||||||
// @block-start snapshot-url
|
|
||||||
const COLLECTION_NAME: &str = "edge-collection";
|
const COLLECTION_NAME: &str = "edge-collection";
|
||||||
|
const SHARD_DIRECTORY: &str = "./qdrant-edge-directory";
|
||||||
|
|
||||||
|
use std::path::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
let snapshot_url = format!(
|
let snapshot_url = format!(
|
||||||
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot"
|
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot"
|
||||||
);
|
);
|
||||||
// @block-end snapshot-url
|
|
||||||
|
|
||||||
// @block-start restore-snapshot
|
|
||||||
const SHARD_DIRECTORY: &str = "./qdrant-edge-directory";
|
|
||||||
let data_dir = Path::new(SHARD_DIRECTORY);
|
let data_dir = Path::new(SHARD_DIRECTORY);
|
||||||
|
|
||||||
let restore_dir = tempfile::Builder::new()
|
let restore_dir = tempfile::Builder::new()
|
||||||
@@ -59,6 +43,9 @@ pub async fn main() -> anyhow::Result<()> {
|
|||||||
// @block-end restore-snapshot
|
// @block-end restore-snapshot
|
||||||
|
|
||||||
// @block-start update-from-snapshot
|
// @block-start update-from-snapshot
|
||||||
|
use qdrant_edge::*;
|
||||||
|
use qdrant_edge::internal::*;
|
||||||
|
|
||||||
let current_manifest = edge_shard.snapshot_manifest()?;
|
let current_manifest = edge_shard.snapshot_manifest()?;
|
||||||
|
|
||||||
let update_url = format!(
|
let update_url = format!(
|
||||||
@@ -96,6 +83,9 @@ pub async fn main() -> anyhow::Result<()> {
|
|||||||
// @block-end update-from-snapshot
|
// @block-end update-from-snapshot
|
||||||
|
|
||||||
// @block-start initialize-edge-shard
|
// @block-start initialize-edge-shard
|
||||||
|
use std::path::*;
|
||||||
|
use qdrant_edge::*;
|
||||||
|
|
||||||
const VECTOR_DIMENSION: usize = 4;
|
const VECTOR_DIMENSION: usize = 4;
|
||||||
const VECTOR_NAME: &str = "my-vector";
|
const VECTOR_NAME: &str = "my-vector";
|
||||||
|
|
||||||
@@ -117,6 +107,10 @@ pub async fn main() -> anyhow::Result<()> {
|
|||||||
// @block-end initialize-edge-shard
|
// @block-end initialize-edge-shard
|
||||||
|
|
||||||
// @block-start initialize-server-client
|
// @block-start initialize-server-client
|
||||||
|
use ::qdrant_client::*;
|
||||||
|
use ::qdrant_client::qdrant::*;
|
||||||
|
use ::qdrant_client::qdrant::Distance;
|
||||||
|
|
||||||
let server_client = Qdrant::from_url(QDRANT_URL)
|
let server_client = Qdrant::from_url(QDRANT_URL)
|
||||||
.api_key(QDRANT_API_KEY)
|
.api_key(QDRANT_API_KEY)
|
||||||
.build()?;
|
.build()?;
|
||||||
@@ -136,19 +130,27 @@ pub async fn main() -> anyhow::Result<()> {
|
|||||||
// @block-end initialize-server-client
|
// @block-end initialize-server-client
|
||||||
|
|
||||||
// @block-start create-upload-queue
|
// @block-start create-upload-queue
|
||||||
|
use std::collections::*;
|
||||||
|
|
||||||
// This is an in-memory queue.
|
// This is an in-memory queue.
|
||||||
// For production use cases consider persisting changes.
|
// For production use cases consider persisting changes.
|
||||||
let mut upload_queue: std::collections::VecDeque<PointStruct> =
|
let mut upload_queue: VecDeque<::qdrant_client::qdrant::PointStruct> = VecDeque::new();
|
||||||
std::collections::VecDeque::new();
|
|
||||||
// @block-end create-upload-queue
|
// @block-end create-upload-queue
|
||||||
|
|
||||||
// @block-start upsert-point
|
// @block-start upsert-point
|
||||||
|
use std::collections::*;
|
||||||
|
use serde_json::json;
|
||||||
|
use qdrant_edge::{
|
||||||
|
PointInsertOperations, PointOperations,
|
||||||
|
PointStructPersisted, PointId, UpdateOperation, Vectors,
|
||||||
|
};
|
||||||
|
|
||||||
let id = 1u64;
|
let id = 1u64;
|
||||||
let vector = vec![0.1f32, 0.2, 0.3, 0.4];
|
let vector = vec![0.1f32, 0.2, 0.3, 0.4];
|
||||||
let payload = json!({"color": "red"});
|
let payload = json!({"color": "red"});
|
||||||
|
|
||||||
let edge_points: Vec<PointStructPersisted> = vec![
|
let edge_points: Vec<PointStructPersisted> = vec![
|
||||||
EdgePoint::new(
|
qdrant_edge::PointStruct::new(
|
||||||
PointId::NumId(id),
|
PointId::NumId(id),
|
||||||
Vectors::new_named([(VECTOR_NAME, vector.clone())]),
|
Vectors::new_named([(VECTOR_NAME, vector.clone())]),
|
||||||
payload.clone(),
|
payload.clone(),
|
||||||
@@ -161,7 +163,7 @@ pub async fn main() -> anyhow::Result<()> {
|
|||||||
),
|
),
|
||||||
))?;
|
))?;
|
||||||
|
|
||||||
let server_point = PointStruct::new(
|
let server_point = ::qdrant_client::qdrant::PointStruct::new(
|
||||||
id,
|
id,
|
||||||
HashMap::from([(VECTOR_NAME.to_string(), vector)]),
|
HashMap::from([(VECTOR_NAME.to_string(), vector)]),
|
||||||
payload.as_object().cloned().unwrap_or_default(),
|
payload.as_object().cloned().unwrap_or_default(),
|
||||||
@@ -170,8 +172,10 @@ pub async fn main() -> anyhow::Result<()> {
|
|||||||
// @block-end upsert-point
|
// @block-end upsert-point
|
||||||
|
|
||||||
// @block-start process-upload-queue
|
// @block-start process-upload-queue
|
||||||
|
use ::qdrant_client::qdrant::*;
|
||||||
|
|
||||||
const BATCH_SIZE: usize = 10;
|
const BATCH_SIZE: usize = 10;
|
||||||
let points_to_upload: Vec<PointStruct> = upload_queue
|
let points_to_upload: Vec<::qdrant_client::qdrant::PointStruct> = upload_queue
|
||||||
.drain(..BATCH_SIZE.min(upload_queue.len()))
|
.drain(..BATCH_SIZE.min(upload_queue.len()))
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user