Edge v0.6 docs (#2199)

* Initial Rust snippets

* Update content for Rust

* Upgrade Rust to 1.94.0; test Edge code snippets against dev

* Temporarily test against Edge shim crate

* Move to EdgeShardConfig and EdgeVectorParams

* Add docs for payload indexing, filtering, facet(), and optimize()

* De-emphasize on-device use case

* Update link to Rust examples on Github

* Flattened API

* Switch to new/create and load to initialize shards

* Fixups for released packages

* Mention recover_partial_snapshot on method list

* Link to Github dev branch for examples

---------

Co-authored-by: xzfc <xzfcpw@gmail.com>
This commit is contained in:
Abdon Pijpelink
2026-03-23 15:10:51 +01:00
committed by GitHub
co-authored by xzfc
parent 2a0ed4346f
commit ea0de4c12e
67 changed files with 5607 additions and 382 deletions
@@ -4,7 +4,7 @@ from qdrant_edge import (
Distance,
EdgeConfig,
EdgeShard,
VectorDataConfig,
EdgeVectorParams,
)
SHARD_DIRECTORY = "./qdrant-edge-directory"
@@ -13,13 +13,13 @@ VECTOR_NAME="my-vector"
Path(SHARD_DIRECTORY).mkdir(parents=True, exist_ok=True)
config = EdgeConfig(
vector_data={
VECTOR_NAME: VectorDataConfig(
vectors={
VECTOR_NAME: EdgeVectorParams(
size=VECTOR_DIMENSION,
distance=Distance.Cosine,
)
}
)
edge_shard = EdgeShard(SHARD_DIRECTORY, config)
edge_shard = EdgeShard.create(SHARD_DIRECTORY, config)
```
@@ -20,7 +20,7 @@ from qdrant_edge import (
Distance,
EdgeConfig,
EdgeShard,
VectorDataConfig,
EdgeVectorParams,
)
SHARD_DIRECTORY = "./qdrant-edge-directory"
@@ -29,15 +29,15 @@ VECTOR_NAME="my-vector"
Path(SHARD_DIRECTORY).mkdir(parents=True, exist_ok=True)
config = EdgeConfig(
vector_data={
VECTOR_NAME: VectorDataConfig(
vectors={
VECTOR_NAME: EdgeVectorParams(
size=VECTOR_DIMENSION,
distance=Distance.Cosine,
)
}
)
edge_shard = EdgeShard(SHARD_DIRECTORY, config)
edge_shard = EdgeShard.create(SHARD_DIRECTORY, config)
from pathlib import Path
from qdrant_edge import Point, UpdateOperation
@@ -22,7 +22,7 @@ from qdrant_edge import (
Distance,
EdgeConfig,
EdgeShard,
VectorDataConfig,
EdgeVectorParams,
)
SHARD_DIRECTORY = "./qdrant-edge-directory"
@@ -31,15 +31,15 @@ VECTOR_NAME="my-vector"
Path(SHARD_DIRECTORY).mkdir(parents=True, exist_ok=True)
config = EdgeConfig(
vector_data={
VECTOR_NAME: VectorDataConfig(
vectors={
VECTOR_NAME: EdgeVectorParams(
size=VECTOR_DIMENSION,
distance=Distance.Cosine,
)
}
)
edge_shard = EdgeShard(SHARD_DIRECTORY, config)
edge_shard = EdgeShard.create(SHARD_DIRECTORY, config)
# @block-end initialize-edge-shard
# @block-start embed-and-store-image
@@ -0,0 +1,3 @@
```rust
drop(edge_shard);
```
@@ -2,15 +2,15 @@
from qdrant_edge import (
Distance,
EdgeConfig,
VectorDataConfig,
EdgeVectorParams,
)
VECTOR_NAME="my-vector"
VECTOR_DIMENSION=4
config = EdgeConfig(
vector_data={
VECTOR_NAME: VectorDataConfig(
vectors={
VECTOR_NAME: EdgeVectorParams(
size=VECTOR_DIMENSION,
distance=Distance.Cosine,
)
@@ -0,0 +1,24 @@
```rust
const VECTOR_NAME: &str = "my-vector";
const VECTOR_DIMENSION: usize = 4;
let config = EdgeConfig {
on_disk_payload: true,
vectors: HashMap::from([(
VECTOR_NAME.to_string(),
EdgeVectorParams {
size: VECTOR_DIMENSION,
distance: Distance::Cosine,
on_disk: Some(true),
quantization_config: None,
multivector_config: None,
datatype: None,
hnsw_config: None,
},
)]),
sparse_vectors: HashMap::new(),
hnsw_config: Default::default(),
quantization_config: None,
optimizers: Default::default(),
};
```
@@ -0,0 +1,17 @@
```python
from qdrant_edge import EdgeOptimizersConfig
config = EdgeConfig(
vectors={
VECTOR_NAME: EdgeVectorParams(
size=VECTOR_DIMENSION,
distance=Distance.Cosine,
)
},
optimizers=EdgeOptimizersConfig(
deleted_threshold=0.2,
vacuum_min_vector_number=100,
default_segment_number=2,
),
)
```
@@ -0,0 +1,26 @@
```rust
let config = EdgeConfig {
on_disk_payload: true,
vectors: HashMap::from([(
VECTOR_NAME.to_string(),
EdgeVectorParams {
size: VECTOR_DIMENSION,
distance: Distance::Cosine,
on_disk: Some(true),
quantization_config: None,
multivector_config: None,
datatype: None,
hnsw_config: None,
},
)]),
sparse_vectors: HashMap::new(),
hnsw_config: Default::default(),
quantization_config: None,
optimizers: EdgeOptimizersConfig {
deleted_threshold: Some(0.2),
vacuum_min_vector_number: Some(100),
default_segment_number: Some(2),
..Default::default()
},
};
```
@@ -0,0 +1,5 @@
```python
from qdrant_edge import PayloadSchemaType
edge_shard.update(UpdateOperation.create_field_index("color", PayloadSchemaType.Keyword))
```
@@ -0,0 +1,10 @@
```rust
edge_shard.update(UpdateOperation::FieldIndexOperation(
FieldIndexOperations::CreateIndex(CreateIndex {
field_name: "color".try_into().unwrap(),
field_schema: Some(PayloadFieldSchema::FieldType(
PayloadSchemaType::Keyword,
)),
}),
))?;
```
@@ -0,0 +1,5 @@
```rust
const SHARD_DIRECTORY: &str = "./qdrant-edge-directory";
fs_err::create_dir_all(SHARD_DIRECTORY)?;
```
@@ -0,0 +1,5 @@
```python
from qdrant_edge import FacetRequest
facet_response = edge_shard.facet(FacetRequest(key="color", limit=10, exact=False))
```
@@ -0,0 +1,8 @@
```rust
let facet_response = edge_shard.facet(FacetRequest {
key: "color".try_into().unwrap(),
limit: 10,
filter: None,
exact: false,
})?;
```
@@ -0,0 +1,20 @@
```python
from qdrant_edge import FieldCondition, Filter, MatchValue
results = edge_shard.query(
QueryRequest(
query=Query.Nearest([0.2, 0.1, 0.9, 0.7], using=VECTOR_NAME),
filter=Filter(
must=[
FieldCondition(
key="color",
match=MatchValue(value="red"),
)
]
),
limit=10,
with_vector=False,
with_payload=True
)
)
```
@@ -0,0 +1,28 @@
```rust
let filter = Filter {
should: None,
min_should: None,
must: Some(vec![Condition::Field(FieldCondition::new_match(
"color".try_into().unwrap(),
Match::Value(MatchValue {
value: ValueVariants::String("red".to_string()),
}),
))]),
must_not: None,
};
let results = edge_shard.query(QueryRequest {
prefetches: vec![],
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery {
query: vec![0.2f32, 0.1, 0.9, 0.7].into(),
using: Some(VECTOR_NAME.to_string()),
}))),
filter: Some(filter),
score_threshold: None,
limit: 10,
offset: 0,
params: None,
with_vector: WithVector::Bool(false),
with_payload: WithPayloadInterface::Bool(true),
})?;
```
@@ -1,5 +1,5 @@
```python
from qdrant_edge import EdgeShard
edge_shard = EdgeShard(SHARD_DIRECTORY, config)
edge_shard = EdgeShard.create(SHARD_DIRECTORY, config)
```
@@ -0,0 +1,6 @@
```rust
let edge_shard = EdgeShard::new(
Path::new(SHARD_DIRECTORY),
config,
)?;
```
@@ -1,3 +1,3 @@
```python
edge_shard = EdgeShard(SHARD_DIRECTORY)
edge_shard = EdgeShard.load(SHARD_DIRECTORY)
```
@@ -0,0 +1,3 @@
```rust
let edge_shard = EdgeShard::load(Path::new(SHARD_DIRECTORY), None)?;
```
@@ -0,0 +1,3 @@
```python
edge_shard.optimize()
```
@@ -0,0 +1,3 @@
```rust
edge_shard.optimize()?;
```
@@ -8,15 +8,15 @@ Path(SHARD_DIRECTORY).mkdir(parents=True, exist_ok=True)
from qdrant_edge import (
Distance,
EdgeConfig,
VectorDataConfig,
EdgeVectorParams,
)
VECTOR_NAME="my-vector"
VECTOR_DIMENSION=4
config = EdgeConfig(
vector_data={
VECTOR_NAME: VectorDataConfig(
vectors={
VECTOR_NAME: EdgeVectorParams(
size=VECTOR_DIMENSION,
distance=Distance.Cosine,
)
@@ -25,7 +25,7 @@ config = EdgeConfig(
from qdrant_edge import EdgeShard
edge_shard = EdgeShard(SHARD_DIRECTORY, config)
edge_shard = EdgeShard.create(SHARD_DIRECTORY, config)
from qdrant_edge import ( Point, UpdateOperation )
@@ -37,7 +37,7 @@ point = Point(
edge_shard.update(UpdateOperation.upsert_points([point]))
point = edge_shard.retrieve(
records = edge_shard.retrieve(
point_ids=[1],
with_payload=True,
with_vector=False
@@ -54,7 +54,52 @@ results = edge_shard.query(
)
)
from qdrant_edge import FieldCondition, Filter, MatchValue
results = edge_shard.query(
QueryRequest(
query=Query.Nearest([0.2, 0.1, 0.9, 0.7], using=VECTOR_NAME),
filter=Filter(
must=[
FieldCondition(
key="color",
match=MatchValue(value="red"),
)
]
),
limit=10,
with_vector=False,
with_payload=True
)
)
from qdrant_edge import FacetRequest
facet_response = edge_shard.facet(FacetRequest(key="color", limit=10, exact=False))
edge_shard.optimize()
from qdrant_edge import EdgeOptimizersConfig
config = EdgeConfig(
vectors={
VECTOR_NAME: EdgeVectorParams(
size=VECTOR_DIMENSION,
distance=Distance.Cosine,
)
},
optimizers=EdgeOptimizersConfig(
deleted_threshold=0.2,
vacuum_min_vector_number=100,
default_segment_number=2,
),
)
from qdrant_edge import PayloadSchemaType
edge_shard.update(UpdateOperation.create_field_index("color", PayloadSchemaType.Keyword))
edge_shard.close()
edge_shard = EdgeShard(SHARD_DIRECTORY)
edge_shard = EdgeShard.load(SHARD_DIRECTORY)
```
@@ -0,0 +1,16 @@
```rust
let results = edge_shard.query(QueryRequest {
prefetches: vec![],
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery {
query: vec![0.2f32, 0.1, 0.9, 0.7].into(),
using: Some(VECTOR_NAME.to_string()),
}))),
filter: None,
score_threshold: None,
limit: 10,
offset: 0,
params: None,
with_vector: WithVector::Bool(false),
with_payload: WithPayloadInterface::Bool(true),
})?;
```
@@ -1,5 +1,5 @@
```python
point = edge_shard.retrieve(
records = edge_shard.retrieve(
point_ids=[1],
with_payload=True,
with_vector=False
@@ -0,0 +1,7 @@
```rust
let retrieved = edge_shard.retrieve(
&[PointId::NumId(1)],
Some(WithPayloadInterface::Bool(true)),
Some(WithVector::Bool(false)),
)?;
```
@@ -0,0 +1,157 @@
```rust
use std::collections::HashMap;
use std::path::Path;
use qdrant_edge::EdgeShard;
use qdrant_edge::{
Condition, CreateIndex, Distance, EdgeConfig, EdgeOptimizersConfig,
EdgeVectorParams, FacetRequest, FieldCondition, FieldIndexOperations,
Filter, Match, MatchValue, NamedQuery, PayloadFieldSchema,
PayloadSchemaType, PointId, PointInsertOperations, PointOperations,
PointStruct, PointStructPersisted, QueryEnum, QueryRequest, ScoringQuery,
UpdateOperation, ValueVariants, Vectors, WithPayloadInterface, WithVector,
};
use serde_json::json;
const SHARD_DIRECTORY: &str = "./qdrant-edge-directory";
fs_err::create_dir_all(SHARD_DIRECTORY)?;
const VECTOR_NAME: &str = "my-vector";
const VECTOR_DIMENSION: usize = 4;
let config = EdgeConfig {
on_disk_payload: true,
vectors: HashMap::from([(
VECTOR_NAME.to_string(),
EdgeVectorParams {
size: VECTOR_DIMENSION,
distance: Distance::Cosine,
on_disk: Some(true),
quantization_config: None,
multivector_config: None,
datatype: None,
hnsw_config: None,
},
)]),
sparse_vectors: HashMap::new(),
hnsw_config: Default::default(),
quantization_config: None,
optimizers: Default::default(),
};
let edge_shard = EdgeShard::new(
Path::new(SHARD_DIRECTORY),
config,
)?;
let points: Vec<PointStructPersisted> = vec![
PointStruct::new(
1u64,
Vectors::new_named([(VECTOR_NAME, vec![0.1f32, 0.2, 0.3, 0.4])]),
json!({"color": "red"}),
)
.into(),
];
edge_shard.update(UpdateOperation::PointOperation(
PointOperations::UpsertPoints(
PointInsertOperations::PointsList(points),
),
))?;
let retrieved = edge_shard.retrieve(
&[PointId::NumId(1)],
Some(WithPayloadInterface::Bool(true)),
Some(WithVector::Bool(false)),
)?;
let results = edge_shard.query(QueryRequest {
prefetches: vec![],
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery {
query: vec![0.2f32, 0.1, 0.9, 0.7].into(),
using: Some(VECTOR_NAME.to_string()),
}))),
filter: None,
score_threshold: None,
limit: 10,
offset: 0,
params: None,
with_vector: WithVector::Bool(false),
with_payload: WithPayloadInterface::Bool(true),
})?;
let filter = Filter {
should: None,
min_should: None,
must: Some(vec![Condition::Field(FieldCondition::new_match(
"color".try_into().unwrap(),
Match::Value(MatchValue {
value: ValueVariants::String("red".to_string()),
}),
))]),
must_not: None,
};
let results = edge_shard.query(QueryRequest {
prefetches: vec![],
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery {
query: vec![0.2f32, 0.1, 0.9, 0.7].into(),
using: Some(VECTOR_NAME.to_string()),
}))),
filter: Some(filter),
score_threshold: None,
limit: 10,
offset: 0,
params: None,
with_vector: WithVector::Bool(false),
with_payload: WithPayloadInterface::Bool(true),
})?;
let facet_response = edge_shard.facet(FacetRequest {
key: "color".try_into().unwrap(),
limit: 10,
filter: None,
exact: false,
})?;
edge_shard.optimize()?;
let config = EdgeConfig {
on_disk_payload: true,
vectors: HashMap::from([(
VECTOR_NAME.to_string(),
EdgeVectorParams {
size: VECTOR_DIMENSION,
distance: Distance::Cosine,
on_disk: Some(true),
quantization_config: None,
multivector_config: None,
datatype: None,
hnsw_config: None,
},
)]),
sparse_vectors: HashMap::new(),
hnsw_config: Default::default(),
quantization_config: None,
optimizers: EdgeOptimizersConfig {
deleted_threshold: Some(0.2),
vacuum_min_vector_number: Some(100),
default_segment_number: Some(2),
..Default::default()
},
};
edge_shard.update(UpdateOperation::FieldIndexOperation(
FieldIndexOperations::CreateIndex(CreateIndex {
field_name: "color".try_into().unwrap(),
field_schema: Some(PayloadFieldSchema::FieldType(
PayloadSchemaType::Keyword,
)),
}),
))?;
drop(edge_shard);
let edge_shard = EdgeShard::load(Path::new(SHARD_DIRECTORY), None)?;
```
@@ -0,0 +1,16 @@
```rust
let points: Vec<PointStructPersisted> = vec![
PointStruct::new(
1u64,
Vectors::new_named([(VECTOR_NAME, vec![0.1f32, 0.2, 0.3, 0.4])]),
json!({"color": "red"}),
)
.into(),
];
edge_shard.update(UpdateOperation::PointOperation(
PointOperations::UpsertPoints(
PointInsertOperations::PointsList(points),
),
))?;
```
@@ -10,15 +10,15 @@ Path(SHARD_DIRECTORY).mkdir(parents=True, exist_ok=True)
from qdrant_edge import (
Distance,
EdgeConfig,
VectorDataConfig,
EdgeVectorParams,
)
VECTOR_NAME="my-vector"
VECTOR_DIMENSION=4
config = EdgeConfig(
vector_data={
VECTOR_NAME: VectorDataConfig(
vectors={
VECTOR_NAME: EdgeVectorParams(
size=VECTOR_DIMENSION,
distance=Distance.Cosine,
)
@@ -29,7 +29,7 @@ config = EdgeConfig(
# @block-start initialize-edge-shard
from qdrant_edge import EdgeShard
edge_shard = EdgeShard(SHARD_DIRECTORY, config)
edge_shard = EdgeShard.create(SHARD_DIRECTORY, config)
# @block-end initialize-edge-shard
# @block-start upsert-points
@@ -45,7 +45,7 @@ edge_shard.update(UpdateOperation.upsert_points([point]))
# @block-end upsert-points
# @block-start retrieve-point
point = edge_shard.retrieve(
records = edge_shard.retrieve(
point_ids=[1],
with_payload=True,
with_vector=False
@@ -65,10 +65,65 @@ results = edge_shard.query(
)
# @block-end query-points
# @block-start filter
from qdrant_edge import FieldCondition, Filter, MatchValue
results = edge_shard.query(
QueryRequest(
query=Query.Nearest([0.2, 0.1, 0.9, 0.7], using=VECTOR_NAME),
filter=Filter(
must=[
FieldCondition(
key="color",
match=MatchValue(value="red"),
)
]
),
limit=10,
with_vector=False,
with_payload=True
)
)
# @block-end filter
# @block-start facet
from qdrant_edge import FacetRequest
facet_response = edge_shard.facet(FacetRequest(key="color", limit=10, exact=False))
# @block-end facet
# @block-start optimize
edge_shard.optimize()
# @block-end optimize
# @block-start configure-optimizer
from qdrant_edge import EdgeOptimizersConfig
config = EdgeConfig(
vectors={
VECTOR_NAME: EdgeVectorParams(
size=VECTOR_DIMENSION,
distance=Distance.Cosine,
)
},
optimizers=EdgeOptimizersConfig(
deleted_threshold=0.2,
vacuum_min_vector_number=100,
default_segment_number=2,
),
)
# @block-end configure-optimizer
# @block-start create-payload-index
from qdrant_edge import PayloadSchemaType
edge_shard.update(UpdateOperation.create_field_index("color", PayloadSchemaType.Keyword))
# @block-end create-payload-index
# @block-start close-edge-shard
edge_shard.close()
# @block-end close-edge-shard
# @block-start load-edge-shard
edge_shard = EdgeShard(SHARD_DIRECTORY)
edge_shard = EdgeShard.load(SHARD_DIRECTORY)
# @block-end load-edge-shard
@@ -0,0 +1,185 @@
use std::collections::HashMap;
use std::path::Path;
use qdrant_edge::EdgeShard;
use qdrant_edge::{
Condition, CreateIndex, Distance, EdgeConfig, EdgeOptimizersConfig,
EdgeVectorParams, FacetRequest, FieldCondition, FieldIndexOperations,
Filter, Match, MatchValue, NamedQuery, PayloadFieldSchema,
PayloadSchemaType, PointId, PointInsertOperations, PointOperations,
PointStruct, PointStructPersisted, QueryEnum, QueryRequest, ScoringQuery,
UpdateOperation, ValueVariants, Vectors, WithPayloadInterface, WithVector,
};
use serde_json::json;
pub async fn main() -> anyhow::Result<()> {
// @block-start create-storage-directory
const SHARD_DIRECTORY: &str = "./qdrant-edge-directory";
fs_err::create_dir_all(SHARD_DIRECTORY)?;
// @block-end create-storage-directory
// @block-start configure-edge-shard
const VECTOR_NAME: &str = "my-vector";
const VECTOR_DIMENSION: usize = 4;
let config = EdgeConfig {
on_disk_payload: true,
vectors: HashMap::from([(
VECTOR_NAME.to_string(),
EdgeVectorParams {
size: VECTOR_DIMENSION,
distance: Distance::Cosine,
on_disk: Some(true),
quantization_config: None,
multivector_config: None,
datatype: None,
hnsw_config: None,
},
)]),
sparse_vectors: HashMap::new(),
hnsw_config: Default::default(),
quantization_config: None,
optimizers: Default::default(),
};
// @block-end configure-edge-shard
// @block-start initialize-edge-shard
let edge_shard = EdgeShard::new(
Path::new(SHARD_DIRECTORY),
config,
)?;
// @block-end initialize-edge-shard
// @block-start upsert-points
let points: Vec<PointStructPersisted> = vec![
PointStruct::new(
1u64,
Vectors::new_named([(VECTOR_NAME, vec![0.1f32, 0.2, 0.3, 0.4])]),
json!({"color": "red"}),
)
.into(),
];
edge_shard.update(UpdateOperation::PointOperation(
PointOperations::UpsertPoints(
PointInsertOperations::PointsList(points),
),
))?;
// @block-end upsert-points
// @block-start retrieve-point
let retrieved = edge_shard.retrieve(
&[PointId::NumId(1)],
Some(WithPayloadInterface::Bool(true)),
Some(WithVector::Bool(false)),
)?;
// @block-end retrieve-point
// @block-start query-points
let results = edge_shard.query(QueryRequest {
prefetches: vec![],
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery {
query: vec![0.2f32, 0.1, 0.9, 0.7].into(),
using: Some(VECTOR_NAME.to_string()),
}))),
filter: None,
score_threshold: None,
limit: 10,
offset: 0,
params: None,
with_vector: WithVector::Bool(false),
with_payload: WithPayloadInterface::Bool(true),
})?;
// @block-end query-points
// @block-start filter
let filter = Filter {
should: None,
min_should: None,
must: Some(vec![Condition::Field(FieldCondition::new_match(
"color".try_into().unwrap(),
Match::Value(MatchValue {
value: ValueVariants::String("red".to_string()),
}),
))]),
must_not: None,
};
let results = edge_shard.query(QueryRequest {
prefetches: vec![],
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery {
query: vec![0.2f32, 0.1, 0.9, 0.7].into(),
using: Some(VECTOR_NAME.to_string()),
}))),
filter: Some(filter),
score_threshold: None,
limit: 10,
offset: 0,
params: None,
with_vector: WithVector::Bool(false),
with_payload: WithPayloadInterface::Bool(true),
})?;
// @block-end filter
// @block-start facet
let facet_response = edge_shard.facet(FacetRequest {
key: "color".try_into().unwrap(),
limit: 10,
filter: None,
exact: false,
})?;
// @block-end facet
// @block-start optimize
edge_shard.optimize()?;
// @block-end optimize
// @block-start configure-optimizer
let config = EdgeConfig {
on_disk_payload: true,
vectors: HashMap::from([(
VECTOR_NAME.to_string(),
EdgeVectorParams {
size: VECTOR_DIMENSION,
distance: Distance::Cosine,
on_disk: Some(true),
quantization_config: None,
multivector_config: None,
datatype: None,
hnsw_config: None,
},
)]),
sparse_vectors: HashMap::new(),
hnsw_config: Default::default(),
quantization_config: None,
optimizers: EdgeOptimizersConfig {
deleted_threshold: Some(0.2),
vacuum_min_vector_number: Some(100),
default_segment_number: Some(2),
..Default::default()
},
};
// @block-end configure-optimizer
// @block-start create-payload-index
edge_shard.update(UpdateOperation::FieldIndexOperation(
FieldIndexOperations::CreateIndex(CreateIndex {
field_name: "color".try_into().unwrap(),
field_schema: Some(PayloadFieldSchema::FieldType(
PayloadSchemaType::Keyword,
)),
}),
))?;
// @block-end create-payload-index
// @block-start close-edge-shard
drop(edge_shard);
// @block-end close-edge-shard
// @block-start load-edge-shard
let edge_shard = EdgeShard::load(Path::new(SHARD_DIRECTORY), None)?;
// @block-end load-edge-shard
Ok(())
}
@@ -0,0 +1,13 @@
```rust
let filter = Filter::new_must(Condition::Field(FieldCondition::new_range(
SYNC_TIMESTAMP_KEY.parse::<JsonPath>().unwrap(),
Range {
lte: Some(OrderedFloat(sync_timestamp)),
..Default::default()
},
)));
mutable_shard.update(UpdateOperation::PointOperation(
PointOperations::DeletePointsByFilter(filter),
))?;
```
@@ -0,0 +1,34 @@
```rust
const SYNC_TIMESTAMP_KEY: &str = "timestamp";
let id = 2u64;
let vector = vec![0.4f32, 0.3, 0.2, 0.1];
let timestamp = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs_f64();
let payload = json!({
"color": "green",
SYNC_TIMESTAMP_KEY: timestamp,
});
let edge_points: Vec<PointStructPersisted> = vec![
EdgePoint::new(
PointId::NumId(id),
Vectors::new_named([(VECTOR_NAME, vector.clone())]),
payload.clone(),
)
.into(),
];
mutable_shard.update(UpdateOperation::PointOperation(
PointOperations::UpsertPoints(
PointInsertOperations::PointsList(edge_points),
),
))?;
let rest_point = PointStruct::new(
id,
HashMap::from([(VECTOR_NAME.to_string(), vector)]),
payload.as_object().cloned().unwrap_or_default(),
);
upload_queue.push_back(rest_point);
```
@@ -25,5 +25,5 @@ with tempfile.TemporaryDirectory(dir=data_dir.parent) as restore_dir:
EdgeShard.unpack_snapshot(str(snapshot_path), str(data_dir))
immutable_shard = EdgeShard(IMMUTABLE_SHARD_DIR)
immutable_shard = EdgeShard.load(IMMUTABLE_SHARD_DIR)
```
@@ -0,0 +1,33 @@
```rust
const COLLECTION_NAME: &str = "edge-collection";
let snapshot_url = format!(
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot"
);
const IMMUTABLE_SHARD_DIR: &str = "./qdrant-edge-directory/immutable";
let data_dir = Path::new(IMMUTABLE_SHARD_DIR);
let restore_dir = tempfile::Builder::new()
.tempdir_in(data_dir.parent().unwrap_or(Path::new(".")))?;
let snapshot_path = restore_dir.path().join("shard.snapshot");
let mut bytes = Vec::new();
std::io::copy(
&mut ureq::get(&snapshot_url)
.header("api-key", QDRANT_API_KEY)
.call()?
.into_body()
.into_reader(),
&mut bytes,
)?;
fs_err::write(&snapshot_path, &bytes)?;
if data_dir.exists() {
fs_err::remove_dir_all(data_dir)?;
}
fs_err::create_dir_all(data_dir)?;
EdgeShard::unpack_snapshot(&snapshot_path, data_dir)?;
let immutable_shard = EdgeShard::load(data_dir, None)?;
```
@@ -4,7 +4,7 @@ from qdrant_edge import (
Distance,
EdgeConfig,
EdgeShard,
VectorDataConfig,
EdgeVectorParams,
)
MUTABLE_SHARD_DIR = "./qdrant-edge-directory/mutable"
@@ -15,13 +15,13 @@ VECTOR_NAME="my-vector"
VECTOR_DIMENSION=4
config = EdgeConfig(
vector_data={
VECTOR_NAME: VectorDataConfig(
vectors={
VECTOR_NAME: EdgeVectorParams(
size=VECTOR_DIMENSION,
distance=Distance.Cosine,
)
}
)
mutable_shard = EdgeShard(MUTABLE_SHARD_DIR, config)
mutable_shard = EdgeShard.create(MUTABLE_SHARD_DIR, config)
```
@@ -0,0 +1,31 @@
```rust
const MUTABLE_SHARD_DIR: &str = "./qdrant-edge-directory/mutable";
const VECTOR_DIMENSION: usize = 4;
const VECTOR_NAME: &str = "my-vector";
fs_err::create_dir_all(MUTABLE_SHARD_DIR)?;
let config = EdgeConfig {
on_disk_payload: true,
vectors: HashMap::from([(
VECTOR_NAME.to_string(),
EdgeVectorParams {
size: VECTOR_DIMENSION,
distance: Distance::Cosine,
on_disk: Some(true),
quantization_config: None,
multivector_config: None,
datatype: None,
hnsw_config: None,
},
)]),
sparse_vectors: HashMap::new(),
hnsw_config: Default::default(),
quantization_config: None,
optimizers: Default::default(),
};
let mutable_shard = EdgeShard::new(
Path::new(MUTABLE_SHARD_DIR),
config,
)?;
```
@@ -4,7 +4,7 @@ from qdrant_edge import (
Distance,
EdgeConfig,
EdgeShard,
VectorDataConfig,
EdgeVectorParams,
)
MUTABLE_SHARD_DIR = "./qdrant-edge-directory/mutable"
@@ -15,15 +15,15 @@ VECTOR_NAME="my-vector"
VECTOR_DIMENSION=4
config = EdgeConfig(
vector_data={
VECTOR_NAME: VectorDataConfig(
vectors={
VECTOR_NAME: EdgeVectorParams(
size=VECTOR_DIMENSION,
distance=Distance.Cosine,
)
}
)
mutable_shard = EdgeShard(MUTABLE_SHARD_DIR, config)
mutable_shard = EdgeShard.create(MUTABLE_SHARD_DIR, config)
import requests
import tempfile
@@ -51,7 +51,7 @@ with tempfile.TemporaryDirectory(dir=data_dir.parent) as restore_dir:
EdgeShard.unpack_snapshot(str(snapshot_path), str(data_dir))
immutable_shard = EdgeShard(IMMUTABLE_SHARD_DIR)
immutable_shard = EdgeShard.load(IMMUTABLE_SHARD_DIR)
from qdrant_edge import ( Point, UpdateOperation )
from qdrant_client import models
@@ -0,0 +1,32 @@
```rust
let query = QueryRequest {
prefetches: vec![],
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery {
query: vec![0.2f32, 0.1, 0.9, 0.7].into(),
using: Some(VECTOR_NAME.to_string()),
}))),
filter: None,
score_threshold: None,
limit: 10,
offset: 0,
params: None,
with_vector: WithVector::Bool(false),
with_payload: WithPayloadInterface::Bool(true),
};
let mut all_results = mutable_shard.query(query.clone())?;
all_results.extend(immutable_shard.query(query)?);
all_results.sort_by(|a, b| {
b.score
.partial_cmp(&a.score)
.unwrap_or(std::cmp::Ordering::Equal)
});
let mut seen_ids = std::collections::HashSet::new();
let results: Vec<_> = all_results
.into_iter()
.filter(|p| seen_ids.insert(p.id.clone()))
.take(10)
.collect();
```
@@ -0,0 +1,196 @@
```rust
use std::collections::HashMap;
use std::path::Path;
use std::time::{SystemTime, UNIX_EPOCH};
use ordered_float::OrderedFloat;
use qdrant_client::qdrant::PointStruct;
use qdrant_edge::EdgeShard;
use qdrant_edge::internal::SnapshotManifest;
use qdrant_edge::{
Condition, Distance, EdgeConfig, EdgeVectorParams, FieldCondition, Filter,
JsonPath, NamedQuery, PointId, PointInsertOperations, PointOperations,
PointStruct as EdgePoint, PointStructPersisted, QueryEnum, QueryRequest,
Range, ScoringQuery, UpdateOperation, Vectors, WithPayloadInterface,
WithVector,
};
use serde_json::json;
const MUTABLE_SHARD_DIR: &str = "./qdrant-edge-directory/mutable";
const VECTOR_DIMENSION: usize = 4;
const VECTOR_NAME: &str = "my-vector";
fs_err::create_dir_all(MUTABLE_SHARD_DIR)?;
let config = EdgeConfig {
on_disk_payload: true,
vectors: HashMap::from([(
VECTOR_NAME.to_string(),
EdgeVectorParams {
size: VECTOR_DIMENSION,
distance: Distance::Cosine,
on_disk: Some(true),
quantization_config: None,
multivector_config: None,
datatype: None,
hnsw_config: None,
},
)]),
sparse_vectors: HashMap::new(),
hnsw_config: Default::default(),
quantization_config: None,
optimizers: Default::default(),
};
let mutable_shard = EdgeShard::new(
Path::new(MUTABLE_SHARD_DIR),
config,
)?;
const COLLECTION_NAME: &str = "edge-collection";
let snapshot_url = format!(
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot"
);
const IMMUTABLE_SHARD_DIR: &str = "./qdrant-edge-directory/immutable";
let data_dir = Path::new(IMMUTABLE_SHARD_DIR);
let restore_dir = tempfile::Builder::new()
.tempdir_in(data_dir.parent().unwrap_or(Path::new(".")))?;
let snapshot_path = restore_dir.path().join("shard.snapshot");
let mut bytes = Vec::new();
std::io::copy(
&mut ureq::get(&snapshot_url)
.header("api-key", QDRANT_API_KEY)
.call()?
.into_body()
.into_reader(),
&mut bytes,
)?;
fs_err::write(&snapshot_path, &bytes)?;
if data_dir.exists() {
fs_err::remove_dir_all(data_dir)?;
}
fs_err::create_dir_all(data_dir)?;
EdgeShard::unpack_snapshot(&snapshot_path, data_dir)?;
let immutable_shard = EdgeShard::load(data_dir, None)?;
const SYNC_TIMESTAMP_KEY: &str = "timestamp";
let id = 2u64;
let vector = vec![0.4f32, 0.3, 0.2, 0.1];
let timestamp = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs_f64();
let payload = json!({
"color": "green",
SYNC_TIMESTAMP_KEY: timestamp,
});
let edge_points: Vec<PointStructPersisted> = vec![
EdgePoint::new(
PointId::NumId(id),
Vectors::new_named([(VECTOR_NAME, vector.clone())]),
payload.clone(),
)
.into(),
];
mutable_shard.update(UpdateOperation::PointOperation(
PointOperations::UpsertPoints(
PointInsertOperations::PointsList(edge_points),
),
))?;
let rest_point = PointStruct::new(
id,
HashMap::from([(VECTOR_NAME.to_string(), vector)]),
payload.as_object().cloned().unwrap_or_default(),
);
upload_queue.push_back(rest_point);
let sync_timestamp = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs_f64();
let current_manifest = immutable_shard.snapshot_manifest()?;
let update_url = format!(
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot\
/partial/create"
);
let temp_dir = tempfile::tempdir_in(data_dir)?;
let partial_snapshot_path = temp_dir.path().join("partial.snapshot");
let mut bytes = Vec::new();
std::io::copy(
&mut ureq::post(&update_url)
.header("api-key", QDRANT_API_KEY)
.send_json(&current_manifest)?
.into_body()
.into_reader(),
&mut bytes,
)?;
fs_err::write(&partial_snapshot_path, &bytes)?;
let unpacked_dir = tempfile::tempdir_in(data_dir)?;
EdgeShard::unpack_snapshot(&partial_snapshot_path, unpacked_dir.path())?;
let snapshot_manifest = SnapshotManifest::load_from_snapshot(
unpacked_dir.path(),
None,
)?;
let immutable_shard = EdgeShard::recover_partial_snapshot(
data_dir,
&current_manifest,
unpacked_dir.path(),
&snapshot_manifest,
)?;
let filter = Filter::new_must(Condition::Field(FieldCondition::new_range(
SYNC_TIMESTAMP_KEY.parse::<JsonPath>().unwrap(),
Range {
lte: Some(OrderedFloat(sync_timestamp)),
..Default::default()
},
)));
mutable_shard.update(UpdateOperation::PointOperation(
PointOperations::DeletePointsByFilter(filter),
))?;
let query = QueryRequest {
prefetches: vec![],
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery {
query: vec![0.2f32, 0.1, 0.9, 0.7].into(),
using: Some(VECTOR_NAME.to_string()),
}))),
filter: None,
score_threshold: None,
limit: 10,
offset: 0,
params: None,
with_vector: WithVector::Bool(false),
with_payload: WithPayloadInterface::Bool(true),
};
let mut all_results = mutable_shard.query(query.clone())?;
all_results.extend(immutable_shard.query(query)?);
all_results.sort_by(|a, b| {
b.score
.partial_cmp(&a.score)
.unwrap_or(std::cmp::Ordering::Equal)
});
let mut seen_ids = std::collections::HashSet::new();
let results: Vec<_> = all_results
.into_iter()
.filter(|p| seen_ids.insert(p.id.clone()))
.take(10)
.collect();
```
@@ -0,0 +1,41 @@
```rust
let sync_timestamp = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs_f64();
let current_manifest = immutable_shard.snapshot_manifest()?;
let update_url = format!(
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot\
/partial/create"
);
let temp_dir = tempfile::tempdir_in(data_dir)?;
let partial_snapshot_path = temp_dir.path().join("partial.snapshot");
let mut bytes = Vec::new();
std::io::copy(
&mut ureq::post(&update_url)
.header("api-key", QDRANT_API_KEY)
.send_json(&current_manifest)?
.into_body()
.into_reader(),
&mut bytes,
)?;
fs_err::write(&partial_snapshot_path, &bytes)?;
let unpacked_dir = tempfile::tempdir_in(data_dir)?;
EdgeShard::unpack_snapshot(&partial_snapshot_path, unpacked_dir.path())?;
let snapshot_manifest = SnapshotManifest::load_from_snapshot(
unpacked_dir.path(),
None,
)?;
let immutable_shard = EdgeShard::recover_partial_snapshot(
data_dir,
&current_manifest,
unpacked_dir.path(),
&snapshot_manifest,
)?;
```
@@ -13,7 +13,7 @@ from qdrant_edge import (
Distance,
EdgeConfig,
EdgeShard,
VectorDataConfig,
EdgeVectorParams,
)
MUTABLE_SHARD_DIR = "./qdrant-edge-directory/mutable"
@@ -24,15 +24,15 @@ VECTOR_NAME="my-vector"
VECTOR_DIMENSION=4
config = EdgeConfig(
vector_data={
VECTOR_NAME: VectorDataConfig(
vectors={
VECTOR_NAME: EdgeVectorParams(
size=VECTOR_DIMENSION,
distance=Distance.Cosine,
)
}
)
mutable_shard = EdgeShard(MUTABLE_SHARD_DIR, config)
mutable_shard = EdgeShard.create(MUTABLE_SHARD_DIR, config)
# @block-end initialize-mutable-shard
# @block-start initialize-immutable-shard
@@ -62,7 +62,7 @@ with tempfile.TemporaryDirectory(dir=data_dir.parent) as restore_dir:
EdgeShard.unpack_snapshot(str(snapshot_path), str(data_dir))
immutable_shard = EdgeShard(IMMUTABLE_SHARD_DIR)
immutable_shard = EdgeShard.load(IMMUTABLE_SHARD_DIR)
# @block-end initialize-immutable-shard
# @block-start dual-write
@@ -0,0 +1,217 @@
use std::collections::HashMap;
use std::path::Path;
use std::time::{SystemTime, UNIX_EPOCH};
use ordered_float::OrderedFloat;
use qdrant_client::qdrant::PointStruct;
use qdrant_edge::EdgeShard;
use qdrant_edge::internal::SnapshotManifest;
use qdrant_edge::{
Condition, Distance, EdgeConfig, EdgeVectorParams, FieldCondition, Filter,
JsonPath, NamedQuery, PointId, PointInsertOperations, PointOperations,
PointStruct as EdgePoint, PointStructPersisted, QueryEnum, QueryRequest,
Range, ScoringQuery, UpdateOperation, Vectors, WithPayloadInterface,
WithVector,
};
use serde_json::json;
pub async fn main() -> anyhow::Result<()> {
// @hide-start
const QDRANT_URL: &str = "";
const QDRANT_API_KEY: &str = "";
let mut upload_queue: std::collections::VecDeque<PointStruct> =
std::collections::VecDeque::new();
// @hide-end
// @block-start initialize-mutable-shard
const MUTABLE_SHARD_DIR: &str = "./qdrant-edge-directory/mutable";
const VECTOR_DIMENSION: usize = 4;
const VECTOR_NAME: &str = "my-vector";
fs_err::create_dir_all(MUTABLE_SHARD_DIR)?;
let config = EdgeConfig {
on_disk_payload: true,
vectors: HashMap::from([(
VECTOR_NAME.to_string(),
EdgeVectorParams {
size: VECTOR_DIMENSION,
distance: Distance::Cosine,
on_disk: Some(true),
quantization_config: None,
multivector_config: None,
datatype: None,
hnsw_config: None,
},
)]),
sparse_vectors: HashMap::new(),
hnsw_config: Default::default(),
quantization_config: None,
optimizers: Default::default(),
};
let mutable_shard = EdgeShard::new(
Path::new(MUTABLE_SHARD_DIR),
config,
)?;
// @block-end initialize-mutable-shard
// @block-start initialize-immutable-shard
const COLLECTION_NAME: &str = "edge-collection";
let snapshot_url = format!(
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot"
);
const IMMUTABLE_SHARD_DIR: &str = "./qdrant-edge-directory/immutable";
let data_dir = Path::new(IMMUTABLE_SHARD_DIR);
let restore_dir = tempfile::Builder::new()
.tempdir_in(data_dir.parent().unwrap_or(Path::new(".")))?;
let snapshot_path = restore_dir.path().join("shard.snapshot");
let mut bytes = Vec::new();
std::io::copy(
&mut ureq::get(&snapshot_url)
.header("api-key", QDRANT_API_KEY)
.call()?
.into_body()
.into_reader(),
&mut bytes,
)?;
fs_err::write(&snapshot_path, &bytes)?;
if data_dir.exists() {
fs_err::remove_dir_all(data_dir)?;
}
fs_err::create_dir_all(data_dir)?;
EdgeShard::unpack_snapshot(&snapshot_path, data_dir)?;
let immutable_shard = EdgeShard::load(data_dir, None)?;
// @block-end initialize-immutable-shard
// @block-start dual-write
const SYNC_TIMESTAMP_KEY: &str = "timestamp";
let id = 2u64;
let vector = vec![0.4f32, 0.3, 0.2, 0.1];
let timestamp = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs_f64();
let payload = json!({
"color": "green",
SYNC_TIMESTAMP_KEY: timestamp,
});
let edge_points: Vec<PointStructPersisted> = vec![
EdgePoint::new(
PointId::NumId(id),
Vectors::new_named([(VECTOR_NAME, vector.clone())]),
payload.clone(),
)
.into(),
];
mutable_shard.update(UpdateOperation::PointOperation(
PointOperations::UpsertPoints(
PointInsertOperations::PointsList(edge_points),
),
))?;
let rest_point = PointStruct::new(
id,
HashMap::from([(VECTOR_NAME.to_string(), vector)]),
payload.as_object().cloned().unwrap_or_default(),
);
upload_queue.push_back(rest_point);
// @block-end dual-write
// @block-start update-immutable-shard
let sync_timestamp = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs_f64();
let current_manifest = immutable_shard.snapshot_manifest()?;
let update_url = format!(
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot\
/partial/create"
);
let temp_dir = tempfile::tempdir_in(data_dir)?;
let partial_snapshot_path = temp_dir.path().join("partial.snapshot");
let mut bytes = Vec::new();
std::io::copy(
&mut ureq::post(&update_url)
.header("api-key", QDRANT_API_KEY)
.send_json(&current_manifest)?
.into_body()
.into_reader(),
&mut bytes,
)?;
fs_err::write(&partial_snapshot_path, &bytes)?;
let unpacked_dir = tempfile::tempdir_in(data_dir)?;
EdgeShard::unpack_snapshot(&partial_snapshot_path, unpacked_dir.path())?;
let snapshot_manifest = SnapshotManifest::load_from_snapshot(
unpacked_dir.path(),
None,
)?;
let immutable_shard = EdgeShard::recover_partial_snapshot(
data_dir,
&current_manifest,
unpacked_dir.path(),
&snapshot_manifest,
)?;
// @block-end update-immutable-shard
// @block-start delete-synced-points
let filter = Filter::new_must(Condition::Field(FieldCondition::new_range(
SYNC_TIMESTAMP_KEY.parse::<JsonPath>().unwrap(),
Range {
lte: Some(OrderedFloat(sync_timestamp)),
..Default::default()
},
)));
mutable_shard.update(UpdateOperation::PointOperation(
PointOperations::DeletePointsByFilter(filter),
))?;
// @block-end delete-synced-points
// @block-start query-both-shards
let query = QueryRequest {
prefetches: vec![],
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery {
query: vec![0.2f32, 0.1, 0.9, 0.7].into(),
using: Some(VECTOR_NAME.to_string()),
}))),
filter: None,
score_threshold: None,
limit: 10,
offset: 0,
params: None,
with_vector: WithVector::Bool(false),
with_payload: WithPayloadInterface::Bool(true),
};
let mut all_results = mutable_shard.query(query.clone())?;
all_results.extend(immutable_shard.query(query)?);
all_results.sort_by(|a, b| {
b.score
.partial_cmp(&a.score)
.unwrap_or(std::cmp::Ordering::Equal)
});
let mut seen_ids = std::collections::HashSet::new();
let results: Vec<_> = all_results
.into_iter()
.filter(|p| seen_ids.insert(p.id.clone()))
.take(10)
.collect();
// @block-end query-both-shards
Ok(())
}
@@ -0,0 +1,6 @@
```rust
// This is an in-memory queue.
// For production use cases consider persisting changes.
let mut upload_queue: std::collections::VecDeque<PointStruct> =
std::collections::VecDeque::new();
```
@@ -3,7 +3,7 @@ from pathlib import Path
from qdrant_edge import (
Distance,
EdgeConfig,
VectorDataConfig,
EdgeVectorParams,
)
SHARD_DIRECTORY = "./qdrant-edge-directory"
@@ -12,13 +12,13 @@ VECTOR_DIMENSION=4
Path(SHARD_DIRECTORY).mkdir(parents=True, exist_ok=True)
config = EdgeConfig(
vector_data={
VECTOR_NAME: VectorDataConfig(
vectors={
VECTOR_NAME: EdgeVectorParams(
size=VECTOR_DIMENSION,
distance=Distance.Cosine,
)
}
)
edge_shard = EdgeShard(SHARD_DIRECTORY, config)
edge_shard = EdgeShard.create(SHARD_DIRECTORY, config)
```
@@ -0,0 +1,30 @@
```rust
const VECTOR_DIMENSION: usize = 4;
const VECTOR_NAME: &str = "my-vector";
fs_err::create_dir_all(SHARD_DIRECTORY)?;
let config = EdgeConfig {
on_disk_payload: true,
vectors: HashMap::from([(
VECTOR_NAME.to_string(),
EdgeVectorParams {
size: VECTOR_DIMENSION,
distance: qdrant_edge::Distance::Cosine,
on_disk: Some(true),
quantization_config: None,
multivector_config: None,
datatype: None,
hnsw_config: None,
},
)]),
sparse_vectors: HashMap::new(),
hnsw_config: Default::default(),
quantization_config: None,
optimizers: Default::default(),
};
let edge_shard = EdgeShard::new(
Path::new(SHARD_DIRECTORY),
config,
)?;
```
@@ -0,0 +1,18 @@
```rust
let server_client = Qdrant::from_url(QDRANT_URL)
.api_key(QDRANT_API_KEY)
.build()?;
if !server_client.collection_exists(COLLECTION_NAME).await? {
server_client
.create_collection(
CreateCollectionBuilder::new(COLLECTION_NAME).vectors_config(
VectorParamsBuilder::new(
VECTOR_DIMENSION as u64,
Distance::Cosine,
),
),
)
.await?;
}
```
@@ -0,0 +1,15 @@
```rust
const BATCH_SIZE: usize = 10;
let points_to_upload: Vec<PointStruct> = upload_queue
.drain(..BATCH_SIZE.min(upload_queue.len()))
.collect();
if !points_to_upload.is_empty() {
server_client
.upsert_points(UpsertPointsBuilder::new(
COLLECTION_NAME,
points_to_upload,
))
.await?;
}
```
@@ -27,7 +27,7 @@ with tempfile.TemporaryDirectory(dir=data_dir.parent) as restore_dir:
EdgeShard.unpack_snapshot(str(snapshot_path), str(data_dir))
edge_shard = EdgeShard(SHARD_DIRECTORY)
edge_shard = EdgeShard.load(SHARD_DIRECTORY)
manifest = edge_shard.snapshot_manifest()
@@ -48,7 +48,7 @@ from pathlib import Path
from qdrant_edge import (
Distance,
EdgeConfig,
VectorDataConfig,
EdgeVectorParams,
)
SHARD_DIRECTORY = "./qdrant-edge-directory"
@@ -57,15 +57,15 @@ VECTOR_DIMENSION=4
Path(SHARD_DIRECTORY).mkdir(parents=True, exist_ok=True)
config = EdgeConfig(
vector_data={
VECTOR_NAME: VectorDataConfig(
vectors={
VECTOR_NAME: EdgeVectorParams(
size=VECTOR_DIMENSION,
distance=Distance.Cosine,
)
}
)
edge_shard = EdgeShard(SHARD_DIRECTORY, config)
edge_shard = EdgeShard.create(SHARD_DIRECTORY, config)
from qdrant_client import QdrantClient, models
@@ -23,5 +23,5 @@ with tempfile.TemporaryDirectory(dir=data_dir.parent) as restore_dir:
EdgeShard.unpack_snapshot(str(snapshot_path), str(data_dir))
edge_shard = EdgeShard(SHARD_DIRECTORY)
edge_shard = EdgeShard.load(SHARD_DIRECTORY)
```
@@ -0,0 +1,28 @@
```rust
const SHARD_DIRECTORY: &str = "./qdrant-edge-directory";
let data_dir = Path::new(SHARD_DIRECTORY);
let restore_dir = tempfile::Builder::new()
.tempdir_in(data_dir.parent().unwrap_or(Path::new(".")))?;
let snapshot_path = restore_dir.path().join("shard.snapshot");
let mut bytes = Vec::new();
std::io::copy(
&mut ureq::get(&snapshot_url)
.header("api-key", QDRANT_API_KEY)
.call()?
.into_body()
.into_reader(),
&mut bytes,
)?;
fs_err::write(&snapshot_path, &bytes)?;
if data_dir.exists() {
fs_err::remove_dir_all(data_dir)?;
}
fs_err::create_dir_all(data_dir)?;
EdgeShard::unpack_snapshot(&snapshot_path, data_dir)?;
let edge_shard = EdgeShard::load(data_dir, None)?;
```
@@ -0,0 +1,176 @@
```rust
use std::collections::HashMap;
use std::path::Path;
use qdrant_client::qdrant::{
CreateCollectionBuilder, Distance, PointStruct, UpsertPointsBuilder,
VectorParamsBuilder,
};
use qdrant_client::Qdrant;
use qdrant_edge::EdgeShard;
use qdrant_edge::internal::SnapshotManifest;
use qdrant_edge::{
EdgeConfig, EdgeVectorParams, PointId, PointInsertOperations,
PointOperations, PointStruct as EdgePoint, PointStructPersisted,
UpdateOperation, Vectors,
};
use serde_json::json;
const COLLECTION_NAME: &str = "edge-collection";
let snapshot_url = format!(
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot"
);
const SHARD_DIRECTORY: &str = "./qdrant-edge-directory";
let data_dir = Path::new(SHARD_DIRECTORY);
let restore_dir = tempfile::Builder::new()
.tempdir_in(data_dir.parent().unwrap_or(Path::new(".")))?;
let snapshot_path = restore_dir.path().join("shard.snapshot");
let mut bytes = Vec::new();
std::io::copy(
&mut ureq::get(&snapshot_url)
.header("api-key", QDRANT_API_KEY)
.call()?
.into_body()
.into_reader(),
&mut bytes,
)?;
fs_err::write(&snapshot_path, &bytes)?;
if data_dir.exists() {
fs_err::remove_dir_all(data_dir)?;
}
fs_err::create_dir_all(data_dir)?;
EdgeShard::unpack_snapshot(&snapshot_path, data_dir)?;
let edge_shard = EdgeShard::load(data_dir, None)?;
let current_manifest = edge_shard.snapshot_manifest()?;
let update_url = format!(
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot\
/partial/create"
);
let temp_dir = tempfile::tempdir_in(data_dir)?;
let partial_snapshot_path = temp_dir.path().join("partial.snapshot");
let mut bytes = Vec::new();
std::io::copy(
&mut ureq::post(&update_url)
.header("api-key", QDRANT_API_KEY)
.send_json(&current_manifest)?
.into_body()
.into_reader(),
&mut bytes,
)?;
fs_err::write(&partial_snapshot_path, &bytes)?;
let unpacked_dir = tempfile::tempdir_in(data_dir)?;
EdgeShard::unpack_snapshot(&partial_snapshot_path, unpacked_dir.path())?;
let snapshot_manifest = SnapshotManifest::load_from_snapshot(
unpacked_dir.path(),
None,
)?;
let edge_shard = EdgeShard::recover_partial_snapshot(
data_dir,
&current_manifest,
unpacked_dir.path(),
&snapshot_manifest,
)?;
const VECTOR_DIMENSION: usize = 4;
const VECTOR_NAME: &str = "my-vector";
fs_err::create_dir_all(SHARD_DIRECTORY)?;
let config = EdgeConfig {
on_disk_payload: true,
vectors: HashMap::from([(
VECTOR_NAME.to_string(),
EdgeVectorParams {
size: VECTOR_DIMENSION,
distance: qdrant_edge::Distance::Cosine,
on_disk: Some(true),
quantization_config: None,
multivector_config: None,
datatype: None,
hnsw_config: None,
},
)]),
sparse_vectors: HashMap::new(),
hnsw_config: Default::default(),
quantization_config: None,
optimizers: Default::default(),
};
let edge_shard = EdgeShard::new(
Path::new(SHARD_DIRECTORY),
config,
)?;
let server_client = Qdrant::from_url(QDRANT_URL)
.api_key(QDRANT_API_KEY)
.build()?;
if !server_client.collection_exists(COLLECTION_NAME).await? {
server_client
.create_collection(
CreateCollectionBuilder::new(COLLECTION_NAME).vectors_config(
VectorParamsBuilder::new(
VECTOR_DIMENSION as u64,
Distance::Cosine,
),
),
)
.await?;
}
// This is an in-memory queue.
// For production use cases consider persisting changes.
let mut upload_queue: std::collections::VecDeque<PointStruct> =
std::collections::VecDeque::new();
let id = 1u64;
let vector = vec![0.1f32, 0.2, 0.3, 0.4];
let payload = json!({"color": "red"});
let edge_points: Vec<PointStructPersisted> = vec![
EdgePoint::new(
PointId::NumId(id),
Vectors::new_named([(VECTOR_NAME, vector.clone())]),
payload.clone(),
)
.into(),
];
edge_shard.update(UpdateOperation::PointOperation(
PointOperations::UpsertPoints(
PointInsertOperations::PointsList(edge_points),
),
))?;
let server_point = PointStruct::new(
id,
HashMap::from([(VECTOR_NAME.to_string(), vector)]),
payload.as_object().cloned().unwrap_or_default(),
);
upload_queue.push_back(server_point);
const BATCH_SIZE: usize = 10;
let points_to_upload: Vec<PointStruct> = upload_queue
.drain(..BATCH_SIZE.min(upload_queue.len()))
.collect();
if !points_to_upload.is_empty() {
server_client
.upsert_points(UpsertPointsBuilder::new(
COLLECTION_NAME,
points_to_upload,
))
.await?;
}
```
@@ -0,0 +1,7 @@
```rust
const COLLECTION_NAME: &str = "edge-collection";
let snapshot_url = format!(
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot"
);
```
@@ -0,0 +1,36 @@
```rust
let current_manifest = edge_shard.snapshot_manifest()?;
let update_url = format!(
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot\
/partial/create"
);
let temp_dir = tempfile::tempdir_in(data_dir)?;
let partial_snapshot_path = temp_dir.path().join("partial.snapshot");
let mut bytes = Vec::new();
std::io::copy(
&mut ureq::post(&update_url)
.header("api-key", QDRANT_API_KEY)
.send_json(&current_manifest)?
.into_body()
.into_reader(),
&mut bytes,
)?;
fs_err::write(&partial_snapshot_path, &bytes)?;
let unpacked_dir = tempfile::tempdir_in(data_dir)?;
EdgeShard::unpack_snapshot(&partial_snapshot_path, unpacked_dir.path())?;
let snapshot_manifest = SnapshotManifest::load_from_snapshot(
unpacked_dir.path(),
None,
)?;
let edge_shard = EdgeShard::recover_partial_snapshot(
data_dir,
&current_manifest,
unpacked_dir.path(),
&snapshot_manifest,
)?;
```
@@ -0,0 +1,26 @@
```rust
let id = 1u64;
let vector = vec![0.1f32, 0.2, 0.3, 0.4];
let payload = json!({"color": "red"});
let edge_points: Vec<PointStructPersisted> = vec![
EdgePoint::new(
PointId::NumId(id),
Vectors::new_named([(VECTOR_NAME, vector.clone())]),
payload.clone(),
)
.into(),
];
edge_shard.update(UpdateOperation::PointOperation(
PointOperations::UpsertPoints(
PointInsertOperations::PointsList(edge_points),
),
))?;
let server_point = PointStruct::new(
id,
HashMap::from([(VECTOR_NAME.to_string(), vector)]),
payload.as_object().cloned().unwrap_or_default(),
);
upload_queue.push_back(server_point);
```
@@ -35,7 +35,7 @@ with tempfile.TemporaryDirectory(dir=data_dir.parent) as restore_dir:
EdgeShard.unpack_snapshot(str(snapshot_path), str(data_dir))
edge_shard = EdgeShard(SHARD_DIRECTORY)
edge_shard = EdgeShard.load(SHARD_DIRECTORY)
# @block-end restore-snapshot
# @block-start update-from-snapshot
@@ -60,7 +60,7 @@ from pathlib import Path
from qdrant_edge import (
Distance,
EdgeConfig,
VectorDataConfig,
EdgeVectorParams,
)
SHARD_DIRECTORY = "./qdrant-edge-directory"
@@ -69,15 +69,15 @@ VECTOR_DIMENSION=4
Path(SHARD_DIRECTORY).mkdir(parents=True, exist_ok=True)
config = EdgeConfig(
vector_data={
VECTOR_NAME: VectorDataConfig(
vectors={
VECTOR_NAME: EdgeVectorParams(
size=VECTOR_DIMENSION,
distance=Distance.Cosine,
)
}
)
edge_shard = EdgeShard(SHARD_DIRECTORY, config)
edge_shard = EdgeShard.create(SHARD_DIRECTORY, config)
# @block-end initialize-edge-shard
# @block-start initialize-server-client
@@ -0,0 +1,199 @@
use std::collections::HashMap;
use std::path::Path;
use qdrant_client::qdrant::{
CreateCollectionBuilder, Distance, PointStruct, UpsertPointsBuilder,
VectorParamsBuilder,
};
use qdrant_client::Qdrant;
use qdrant_edge::EdgeShard;
use qdrant_edge::internal::SnapshotManifest;
use qdrant_edge::{
EdgeConfig, EdgeVectorParams, PointId, PointInsertOperations,
PointOperations, PointStruct as EdgePoint, PointStructPersisted,
UpdateOperation, Vectors,
};
use serde_json::json;
pub async fn main() -> anyhow::Result<()> {
// @hide-start
const QDRANT_URL: &str = "";
const QDRANT_API_KEY: &str = "";
// @hide-end
// @block-start snapshot-url
const COLLECTION_NAME: &str = "edge-collection";
let snapshot_url = format!(
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot"
);
// @block-end snapshot-url
// @block-start restore-snapshot
const SHARD_DIRECTORY: &str = "./qdrant-edge-directory";
let data_dir = Path::new(SHARD_DIRECTORY);
let restore_dir = tempfile::Builder::new()
.tempdir_in(data_dir.parent().unwrap_or(Path::new(".")))?;
let snapshot_path = restore_dir.path().join("shard.snapshot");
let mut bytes = Vec::new();
std::io::copy(
&mut ureq::get(&snapshot_url)
.header("api-key", QDRANT_API_KEY)
.call()?
.into_body()
.into_reader(),
&mut bytes,
)?;
fs_err::write(&snapshot_path, &bytes)?;
if data_dir.exists() {
fs_err::remove_dir_all(data_dir)?;
}
fs_err::create_dir_all(data_dir)?;
EdgeShard::unpack_snapshot(&snapshot_path, data_dir)?;
let edge_shard = EdgeShard::load(data_dir, None)?;
// @block-end restore-snapshot
// @block-start update-from-snapshot
let current_manifest = edge_shard.snapshot_manifest()?;
let update_url = format!(
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot\
/partial/create"
);
let temp_dir = tempfile::tempdir_in(data_dir)?;
let partial_snapshot_path = temp_dir.path().join("partial.snapshot");
let mut bytes = Vec::new();
std::io::copy(
&mut ureq::post(&update_url)
.header("api-key", QDRANT_API_KEY)
.send_json(&current_manifest)?
.into_body()
.into_reader(),
&mut bytes,
)?;
fs_err::write(&partial_snapshot_path, &bytes)?;
let unpacked_dir = tempfile::tempdir_in(data_dir)?;
EdgeShard::unpack_snapshot(&partial_snapshot_path, unpacked_dir.path())?;
let snapshot_manifest = SnapshotManifest::load_from_snapshot(
unpacked_dir.path(),
None,
)?;
let edge_shard = EdgeShard::recover_partial_snapshot(
data_dir,
&current_manifest,
unpacked_dir.path(),
&snapshot_manifest,
)?;
// @block-end update-from-snapshot
// @block-start initialize-edge-shard
const VECTOR_DIMENSION: usize = 4;
const VECTOR_NAME: &str = "my-vector";
fs_err::create_dir_all(SHARD_DIRECTORY)?;
let config = EdgeConfig {
on_disk_payload: true,
vectors: HashMap::from([(
VECTOR_NAME.to_string(),
EdgeVectorParams {
size: VECTOR_DIMENSION,
distance: qdrant_edge::Distance::Cosine,
on_disk: Some(true),
quantization_config: None,
multivector_config: None,
datatype: None,
hnsw_config: None,
},
)]),
sparse_vectors: HashMap::new(),
hnsw_config: Default::default(),
quantization_config: None,
optimizers: Default::default(),
};
let edge_shard = EdgeShard::new(
Path::new(SHARD_DIRECTORY),
config,
)?;
// @block-end initialize-edge-shard
// @block-start initialize-server-client
let server_client = Qdrant::from_url(QDRANT_URL)
.api_key(QDRANT_API_KEY)
.build()?;
if !server_client.collection_exists(COLLECTION_NAME).await? {
server_client
.create_collection(
CreateCollectionBuilder::new(COLLECTION_NAME).vectors_config(
VectorParamsBuilder::new(
VECTOR_DIMENSION as u64,
Distance::Cosine,
),
),
)
.await?;
}
// @block-end initialize-server-client
// @block-start create-upload-queue
// This is an in-memory queue.
// For production use cases consider persisting changes.
let mut upload_queue: std::collections::VecDeque<PointStruct> =
std::collections::VecDeque::new();
// @block-end create-upload-queue
// @block-start upsert-point
let id = 1u64;
let vector = vec![0.1f32, 0.2, 0.3, 0.4];
let payload = json!({"color": "red"});
let edge_points: Vec<PointStructPersisted> = vec![
EdgePoint::new(
PointId::NumId(id),
Vectors::new_named([(VECTOR_NAME, vector.clone())]),
payload.clone(),
)
.into(),
];
edge_shard.update(UpdateOperation::PointOperation(
PointOperations::UpsertPoints(
PointInsertOperations::PointsList(edge_points),
),
))?;
let server_point = PointStruct::new(
id,
HashMap::from([(VECTOR_NAME.to_string(), vector)]),
payload.as_object().cloned().unwrap_or_default(),
);
upload_queue.push_back(server_point);
// @block-end upsert-point
// @block-start process-upload-queue
const BATCH_SIZE: usize = 10;
let points_to_upload: Vec<PointStruct> = upload_queue
.drain(..BATCH_SIZE.min(upload_queue.len()))
.collect();
if !points_to_upload.is_empty() {
server_client
.upsert_points(UpsertPointsBuilder::new(
COLLECTION_NAME,
points_to_upload,
))
.await?;
}
// @block-end process-upload-queue
Ok(())
}