mirror of
https://github.com/qdrant/landing_page.git
synced 2026-10-09 21:08:31 +02:00
Edge v0.6 docs (#2199)
* Initial Rust snippets * Update content for Rust * Upgrade Rust to 1.94.0; test Edge code snippets against dev * Temporarily test against Edge shim crate * Move to EdgeShardConfig and EdgeVectorParams * Add docs for payload indexing, filtering, facet(), and optimize() * De-emphasize on-device use case * Update link to Rust examples on Github * Flattened API * Switch to new/create and load to initialize shards * Fixups for released packages * Mention recover_partial_snapshot on method list * Link to Github dev branch for examples --------- Co-authored-by: xzfc <xzfcpw@gmail.com>
This commit is contained in:
+4
-4
@@ -4,7 +4,7 @@ from qdrant_edge import (
|
||||
Distance,
|
||||
EdgeConfig,
|
||||
EdgeShard,
|
||||
VectorDataConfig,
|
||||
EdgeVectorParams,
|
||||
)
|
||||
|
||||
SHARD_DIRECTORY = "./qdrant-edge-directory"
|
||||
@@ -13,13 +13,13 @@ VECTOR_NAME="my-vector"
|
||||
|
||||
Path(SHARD_DIRECTORY).mkdir(parents=True, exist_ok=True)
|
||||
config = EdgeConfig(
|
||||
vector_data={
|
||||
VECTOR_NAME: VectorDataConfig(
|
||||
vectors={
|
||||
VECTOR_NAME: EdgeVectorParams(
|
||||
size=VECTOR_DIMENSION,
|
||||
distance=Distance.Cosine,
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
edge_shard = EdgeShard(SHARD_DIRECTORY, config)
|
||||
edge_shard = EdgeShard.create(SHARD_DIRECTORY, config)
|
||||
```
|
||||
|
||||
+4
-4
@@ -20,7 +20,7 @@ from qdrant_edge import (
|
||||
Distance,
|
||||
EdgeConfig,
|
||||
EdgeShard,
|
||||
VectorDataConfig,
|
||||
EdgeVectorParams,
|
||||
)
|
||||
|
||||
SHARD_DIRECTORY = "./qdrant-edge-directory"
|
||||
@@ -29,15 +29,15 @@ VECTOR_NAME="my-vector"
|
||||
|
||||
Path(SHARD_DIRECTORY).mkdir(parents=True, exist_ok=True)
|
||||
config = EdgeConfig(
|
||||
vector_data={
|
||||
VECTOR_NAME: VectorDataConfig(
|
||||
vectors={
|
||||
VECTOR_NAME: EdgeVectorParams(
|
||||
size=VECTOR_DIMENSION,
|
||||
distance=Distance.Cosine,
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
edge_shard = EdgeShard(SHARD_DIRECTORY, config)
|
||||
edge_shard = EdgeShard.create(SHARD_DIRECTORY, config)
|
||||
|
||||
from pathlib import Path
|
||||
from qdrant_edge import Point, UpdateOperation
|
||||
|
||||
@@ -22,7 +22,7 @@ from qdrant_edge import (
|
||||
Distance,
|
||||
EdgeConfig,
|
||||
EdgeShard,
|
||||
VectorDataConfig,
|
||||
EdgeVectorParams,
|
||||
)
|
||||
|
||||
SHARD_DIRECTORY = "./qdrant-edge-directory"
|
||||
@@ -31,15 +31,15 @@ VECTOR_NAME="my-vector"
|
||||
|
||||
Path(SHARD_DIRECTORY).mkdir(parents=True, exist_ok=True)
|
||||
config = EdgeConfig(
|
||||
vector_data={
|
||||
VECTOR_NAME: VectorDataConfig(
|
||||
vectors={
|
||||
VECTOR_NAME: EdgeVectorParams(
|
||||
size=VECTOR_DIMENSION,
|
||||
distance=Distance.Cosine,
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
edge_shard = EdgeShard(SHARD_DIRECTORY, config)
|
||||
edge_shard = EdgeShard.create(SHARD_DIRECTORY, config)
|
||||
# @block-end initialize-edge-shard
|
||||
|
||||
# @block-start embed-and-store-image
|
||||
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
```rust
|
||||
drop(edge_shard);
|
||||
```
|
||||
+3
-3
@@ -2,15 +2,15 @@
|
||||
from qdrant_edge import (
|
||||
Distance,
|
||||
EdgeConfig,
|
||||
VectorDataConfig,
|
||||
EdgeVectorParams,
|
||||
)
|
||||
|
||||
VECTOR_NAME="my-vector"
|
||||
VECTOR_DIMENSION=4
|
||||
|
||||
config = EdgeConfig(
|
||||
vector_data={
|
||||
VECTOR_NAME: VectorDataConfig(
|
||||
vectors={
|
||||
VECTOR_NAME: EdgeVectorParams(
|
||||
size=VECTOR_DIMENSION,
|
||||
distance=Distance.Cosine,
|
||||
)
|
||||
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
```rust
|
||||
const VECTOR_NAME: &str = "my-vector";
|
||||
const VECTOR_DIMENSION: usize = 4;
|
||||
|
||||
let config = EdgeConfig {
|
||||
on_disk_payload: true,
|
||||
vectors: HashMap::from([(
|
||||
VECTOR_NAME.to_string(),
|
||||
EdgeVectorParams {
|
||||
size: VECTOR_DIMENSION,
|
||||
distance: Distance::Cosine,
|
||||
on_disk: Some(true),
|
||||
quantization_config: None,
|
||||
multivector_config: None,
|
||||
datatype: None,
|
||||
hnsw_config: None,
|
||||
},
|
||||
)]),
|
||||
sparse_vectors: HashMap::new(),
|
||||
hnsw_config: Default::default(),
|
||||
quantization_config: None,
|
||||
optimizers: Default::default(),
|
||||
};
|
||||
```
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
```python
|
||||
from qdrant_edge import EdgeOptimizersConfig
|
||||
|
||||
config = EdgeConfig(
|
||||
vectors={
|
||||
VECTOR_NAME: EdgeVectorParams(
|
||||
size=VECTOR_DIMENSION,
|
||||
distance=Distance.Cosine,
|
||||
)
|
||||
},
|
||||
optimizers=EdgeOptimizersConfig(
|
||||
deleted_threshold=0.2,
|
||||
vacuum_min_vector_number=100,
|
||||
default_segment_number=2,
|
||||
),
|
||||
)
|
||||
```
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
```rust
|
||||
let config = EdgeConfig {
|
||||
on_disk_payload: true,
|
||||
vectors: HashMap::from([(
|
||||
VECTOR_NAME.to_string(),
|
||||
EdgeVectorParams {
|
||||
size: VECTOR_DIMENSION,
|
||||
distance: Distance::Cosine,
|
||||
on_disk: Some(true),
|
||||
quantization_config: None,
|
||||
multivector_config: None,
|
||||
datatype: None,
|
||||
hnsw_config: None,
|
||||
},
|
||||
)]),
|
||||
sparse_vectors: HashMap::new(),
|
||||
hnsw_config: Default::default(),
|
||||
quantization_config: None,
|
||||
optimizers: EdgeOptimizersConfig {
|
||||
deleted_threshold: Some(0.2),
|
||||
vacuum_min_vector_number: Some(100),
|
||||
default_segment_number: Some(2),
|
||||
..Default::default()
|
||||
},
|
||||
};
|
||||
```
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
```python
|
||||
from qdrant_edge import PayloadSchemaType
|
||||
|
||||
edge_shard.update(UpdateOperation.create_field_index("color", PayloadSchemaType.Keyword))
|
||||
```
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
```rust
|
||||
edge_shard.update(UpdateOperation::FieldIndexOperation(
|
||||
FieldIndexOperations::CreateIndex(CreateIndex {
|
||||
field_name: "color".try_into().unwrap(),
|
||||
field_schema: Some(PayloadFieldSchema::FieldType(
|
||||
PayloadSchemaType::Keyword,
|
||||
)),
|
||||
}),
|
||||
))?;
|
||||
```
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
```rust
|
||||
const SHARD_DIRECTORY: &str = "./qdrant-edge-directory";
|
||||
|
||||
fs_err::create_dir_all(SHARD_DIRECTORY)?;
|
||||
```
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
```python
|
||||
from qdrant_edge import FacetRequest
|
||||
|
||||
facet_response = edge_shard.facet(FacetRequest(key="color", limit=10, exact=False))
|
||||
```
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
```rust
|
||||
let facet_response = edge_shard.facet(FacetRequest {
|
||||
key: "color".try_into().unwrap(),
|
||||
limit: 10,
|
||||
filter: None,
|
||||
exact: false,
|
||||
})?;
|
||||
```
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
```python
|
||||
from qdrant_edge import FieldCondition, Filter, MatchValue
|
||||
|
||||
results = edge_shard.query(
|
||||
QueryRequest(
|
||||
query=Query.Nearest([0.2, 0.1, 0.9, 0.7], using=VECTOR_NAME),
|
||||
filter=Filter(
|
||||
must=[
|
||||
FieldCondition(
|
||||
key="color",
|
||||
match=MatchValue(value="red"),
|
||||
)
|
||||
]
|
||||
),
|
||||
limit=10,
|
||||
with_vector=False,
|
||||
with_payload=True
|
||||
)
|
||||
)
|
||||
```
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
```rust
|
||||
let filter = Filter {
|
||||
should: None,
|
||||
min_should: None,
|
||||
must: Some(vec![Condition::Field(FieldCondition::new_match(
|
||||
"color".try_into().unwrap(),
|
||||
Match::Value(MatchValue {
|
||||
value: ValueVariants::String("red".to_string()),
|
||||
}),
|
||||
))]),
|
||||
must_not: None,
|
||||
};
|
||||
|
||||
let results = edge_shard.query(QueryRequest {
|
||||
prefetches: vec![],
|
||||
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery {
|
||||
query: vec![0.2f32, 0.1, 0.9, 0.7].into(),
|
||||
using: Some(VECTOR_NAME.to_string()),
|
||||
}))),
|
||||
filter: Some(filter),
|
||||
score_threshold: None,
|
||||
limit: 10,
|
||||
offset: 0,
|
||||
params: None,
|
||||
with_vector: WithVector::Bool(false),
|
||||
with_payload: WithPayloadInterface::Bool(true),
|
||||
})?;
|
||||
```
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
```python
|
||||
from qdrant_edge import EdgeShard
|
||||
|
||||
edge_shard = EdgeShard(SHARD_DIRECTORY, config)
|
||||
edge_shard = EdgeShard.create(SHARD_DIRECTORY, config)
|
||||
```
|
||||
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
```rust
|
||||
let edge_shard = EdgeShard::new(
|
||||
Path::new(SHARD_DIRECTORY),
|
||||
config,
|
||||
)?;
|
||||
```
|
||||
+1
-1
@@ -1,3 +1,3 @@
|
||||
```python
|
||||
edge_shard = EdgeShard(SHARD_DIRECTORY)
|
||||
edge_shard = EdgeShard.load(SHARD_DIRECTORY)
|
||||
```
|
||||
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
```rust
|
||||
let edge_shard = EdgeShard::load(Path::new(SHARD_DIRECTORY), None)?;
|
||||
```
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
```python
|
||||
edge_shard.optimize()
|
||||
```
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
```rust
|
||||
edge_shard.optimize()?;
|
||||
```
|
||||
+51
-6
@@ -8,15 +8,15 @@ Path(SHARD_DIRECTORY).mkdir(parents=True, exist_ok=True)
|
||||
from qdrant_edge import (
|
||||
Distance,
|
||||
EdgeConfig,
|
||||
VectorDataConfig,
|
||||
EdgeVectorParams,
|
||||
)
|
||||
|
||||
VECTOR_NAME="my-vector"
|
||||
VECTOR_DIMENSION=4
|
||||
|
||||
config = EdgeConfig(
|
||||
vector_data={
|
||||
VECTOR_NAME: VectorDataConfig(
|
||||
vectors={
|
||||
VECTOR_NAME: EdgeVectorParams(
|
||||
size=VECTOR_DIMENSION,
|
||||
distance=Distance.Cosine,
|
||||
)
|
||||
@@ -25,7 +25,7 @@ config = EdgeConfig(
|
||||
|
||||
from qdrant_edge import EdgeShard
|
||||
|
||||
edge_shard = EdgeShard(SHARD_DIRECTORY, config)
|
||||
edge_shard = EdgeShard.create(SHARD_DIRECTORY, config)
|
||||
|
||||
from qdrant_edge import ( Point, UpdateOperation )
|
||||
|
||||
@@ -37,7 +37,7 @@ point = Point(
|
||||
|
||||
edge_shard.update(UpdateOperation.upsert_points([point]))
|
||||
|
||||
point = edge_shard.retrieve(
|
||||
records = edge_shard.retrieve(
|
||||
point_ids=[1],
|
||||
with_payload=True,
|
||||
with_vector=False
|
||||
@@ -54,7 +54,52 @@ results = edge_shard.query(
|
||||
)
|
||||
)
|
||||
|
||||
from qdrant_edge import FieldCondition, Filter, MatchValue
|
||||
|
||||
results = edge_shard.query(
|
||||
QueryRequest(
|
||||
query=Query.Nearest([0.2, 0.1, 0.9, 0.7], using=VECTOR_NAME),
|
||||
filter=Filter(
|
||||
must=[
|
||||
FieldCondition(
|
||||
key="color",
|
||||
match=MatchValue(value="red"),
|
||||
)
|
||||
]
|
||||
),
|
||||
limit=10,
|
||||
with_vector=False,
|
||||
with_payload=True
|
||||
)
|
||||
)
|
||||
|
||||
from qdrant_edge import FacetRequest
|
||||
|
||||
facet_response = edge_shard.facet(FacetRequest(key="color", limit=10, exact=False))
|
||||
|
||||
edge_shard.optimize()
|
||||
|
||||
from qdrant_edge import EdgeOptimizersConfig
|
||||
|
||||
config = EdgeConfig(
|
||||
vectors={
|
||||
VECTOR_NAME: EdgeVectorParams(
|
||||
size=VECTOR_DIMENSION,
|
||||
distance=Distance.Cosine,
|
||||
)
|
||||
},
|
||||
optimizers=EdgeOptimizersConfig(
|
||||
deleted_threshold=0.2,
|
||||
vacuum_min_vector_number=100,
|
||||
default_segment_number=2,
|
||||
),
|
||||
)
|
||||
|
||||
from qdrant_edge import PayloadSchemaType
|
||||
|
||||
edge_shard.update(UpdateOperation.create_field_index("color", PayloadSchemaType.Keyword))
|
||||
|
||||
edge_shard.close()
|
||||
|
||||
edge_shard = EdgeShard(SHARD_DIRECTORY)
|
||||
edge_shard = EdgeShard.load(SHARD_DIRECTORY)
|
||||
```
|
||||
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
```rust
|
||||
let results = edge_shard.query(QueryRequest {
|
||||
prefetches: vec![],
|
||||
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery {
|
||||
query: vec![0.2f32, 0.1, 0.9, 0.7].into(),
|
||||
using: Some(VECTOR_NAME.to_string()),
|
||||
}))),
|
||||
filter: None,
|
||||
score_threshold: None,
|
||||
limit: 10,
|
||||
offset: 0,
|
||||
params: None,
|
||||
with_vector: WithVector::Bool(false),
|
||||
with_payload: WithPayloadInterface::Bool(true),
|
||||
})?;
|
||||
```
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
```python
|
||||
point = edge_shard.retrieve(
|
||||
records = edge_shard.retrieve(
|
||||
point_ids=[1],
|
||||
with_payload=True,
|
||||
with_vector=False
|
||||
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
```rust
|
||||
let retrieved = edge_shard.retrieve(
|
||||
&[PointId::NumId(1)],
|
||||
Some(WithPayloadInterface::Bool(true)),
|
||||
Some(WithVector::Bool(false)),
|
||||
)?;
|
||||
```
|
||||
+157
@@ -0,0 +1,157 @@
|
||||
```rust
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
|
||||
use qdrant_edge::EdgeShard;
|
||||
use qdrant_edge::{
|
||||
Condition, CreateIndex, Distance, EdgeConfig, EdgeOptimizersConfig,
|
||||
EdgeVectorParams, FacetRequest, FieldCondition, FieldIndexOperations,
|
||||
Filter, Match, MatchValue, NamedQuery, PayloadFieldSchema,
|
||||
PayloadSchemaType, PointId, PointInsertOperations, PointOperations,
|
||||
PointStruct, PointStructPersisted, QueryEnum, QueryRequest, ScoringQuery,
|
||||
UpdateOperation, ValueVariants, Vectors, WithPayloadInterface, WithVector,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
const SHARD_DIRECTORY: &str = "./qdrant-edge-directory";
|
||||
|
||||
fs_err::create_dir_all(SHARD_DIRECTORY)?;
|
||||
|
||||
const VECTOR_NAME: &str = "my-vector";
|
||||
const VECTOR_DIMENSION: usize = 4;
|
||||
|
||||
let config = EdgeConfig {
|
||||
on_disk_payload: true,
|
||||
vectors: HashMap::from([(
|
||||
VECTOR_NAME.to_string(),
|
||||
EdgeVectorParams {
|
||||
size: VECTOR_DIMENSION,
|
||||
distance: Distance::Cosine,
|
||||
on_disk: Some(true),
|
||||
quantization_config: None,
|
||||
multivector_config: None,
|
||||
datatype: None,
|
||||
hnsw_config: None,
|
||||
},
|
||||
)]),
|
||||
sparse_vectors: HashMap::new(),
|
||||
hnsw_config: Default::default(),
|
||||
quantization_config: None,
|
||||
optimizers: Default::default(),
|
||||
};
|
||||
|
||||
let edge_shard = EdgeShard::new(
|
||||
Path::new(SHARD_DIRECTORY),
|
||||
config,
|
||||
)?;
|
||||
|
||||
let points: Vec<PointStructPersisted> = vec![
|
||||
PointStruct::new(
|
||||
1u64,
|
||||
Vectors::new_named([(VECTOR_NAME, vec![0.1f32, 0.2, 0.3, 0.4])]),
|
||||
json!({"color": "red"}),
|
||||
)
|
||||
.into(),
|
||||
];
|
||||
|
||||
edge_shard.update(UpdateOperation::PointOperation(
|
||||
PointOperations::UpsertPoints(
|
||||
PointInsertOperations::PointsList(points),
|
||||
),
|
||||
))?;
|
||||
|
||||
let retrieved = edge_shard.retrieve(
|
||||
&[PointId::NumId(1)],
|
||||
Some(WithPayloadInterface::Bool(true)),
|
||||
Some(WithVector::Bool(false)),
|
||||
)?;
|
||||
|
||||
let results = edge_shard.query(QueryRequest {
|
||||
prefetches: vec![],
|
||||
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery {
|
||||
query: vec![0.2f32, 0.1, 0.9, 0.7].into(),
|
||||
using: Some(VECTOR_NAME.to_string()),
|
||||
}))),
|
||||
filter: None,
|
||||
score_threshold: None,
|
||||
limit: 10,
|
||||
offset: 0,
|
||||
params: None,
|
||||
with_vector: WithVector::Bool(false),
|
||||
with_payload: WithPayloadInterface::Bool(true),
|
||||
})?;
|
||||
|
||||
let filter = Filter {
|
||||
should: None,
|
||||
min_should: None,
|
||||
must: Some(vec![Condition::Field(FieldCondition::new_match(
|
||||
"color".try_into().unwrap(),
|
||||
Match::Value(MatchValue {
|
||||
value: ValueVariants::String("red".to_string()),
|
||||
}),
|
||||
))]),
|
||||
must_not: None,
|
||||
};
|
||||
|
||||
let results = edge_shard.query(QueryRequest {
|
||||
prefetches: vec![],
|
||||
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery {
|
||||
query: vec![0.2f32, 0.1, 0.9, 0.7].into(),
|
||||
using: Some(VECTOR_NAME.to_string()),
|
||||
}))),
|
||||
filter: Some(filter),
|
||||
score_threshold: None,
|
||||
limit: 10,
|
||||
offset: 0,
|
||||
params: None,
|
||||
with_vector: WithVector::Bool(false),
|
||||
with_payload: WithPayloadInterface::Bool(true),
|
||||
})?;
|
||||
|
||||
let facet_response = edge_shard.facet(FacetRequest {
|
||||
key: "color".try_into().unwrap(),
|
||||
limit: 10,
|
||||
filter: None,
|
||||
exact: false,
|
||||
})?;
|
||||
|
||||
edge_shard.optimize()?;
|
||||
|
||||
let config = EdgeConfig {
|
||||
on_disk_payload: true,
|
||||
vectors: HashMap::from([(
|
||||
VECTOR_NAME.to_string(),
|
||||
EdgeVectorParams {
|
||||
size: VECTOR_DIMENSION,
|
||||
distance: Distance::Cosine,
|
||||
on_disk: Some(true),
|
||||
quantization_config: None,
|
||||
multivector_config: None,
|
||||
datatype: None,
|
||||
hnsw_config: None,
|
||||
},
|
||||
)]),
|
||||
sparse_vectors: HashMap::new(),
|
||||
hnsw_config: Default::default(),
|
||||
quantization_config: None,
|
||||
optimizers: EdgeOptimizersConfig {
|
||||
deleted_threshold: Some(0.2),
|
||||
vacuum_min_vector_number: Some(100),
|
||||
default_segment_number: Some(2),
|
||||
..Default::default()
|
||||
},
|
||||
};
|
||||
|
||||
edge_shard.update(UpdateOperation::FieldIndexOperation(
|
||||
FieldIndexOperations::CreateIndex(CreateIndex {
|
||||
field_name: "color".try_into().unwrap(),
|
||||
field_schema: Some(PayloadFieldSchema::FieldType(
|
||||
PayloadSchemaType::Keyword,
|
||||
)),
|
||||
}),
|
||||
))?;
|
||||
|
||||
drop(edge_shard);
|
||||
|
||||
let edge_shard = EdgeShard::load(Path::new(SHARD_DIRECTORY), None)?;
|
||||
```
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
```rust
|
||||
let points: Vec<PointStructPersisted> = vec![
|
||||
PointStruct::new(
|
||||
1u64,
|
||||
Vectors::new_named([(VECTOR_NAME, vec![0.1f32, 0.2, 0.3, 0.4])]),
|
||||
json!({"color": "red"}),
|
||||
)
|
||||
.into(),
|
||||
];
|
||||
|
||||
edge_shard.update(UpdateOperation::PointOperation(
|
||||
PointOperations::UpsertPoints(
|
||||
PointInsertOperations::PointsList(points),
|
||||
),
|
||||
))?;
|
||||
```
|
||||
@@ -10,15 +10,15 @@ Path(SHARD_DIRECTORY).mkdir(parents=True, exist_ok=True)
|
||||
from qdrant_edge import (
|
||||
Distance,
|
||||
EdgeConfig,
|
||||
VectorDataConfig,
|
||||
EdgeVectorParams,
|
||||
)
|
||||
|
||||
VECTOR_NAME="my-vector"
|
||||
VECTOR_DIMENSION=4
|
||||
|
||||
config = EdgeConfig(
|
||||
vector_data={
|
||||
VECTOR_NAME: VectorDataConfig(
|
||||
vectors={
|
||||
VECTOR_NAME: EdgeVectorParams(
|
||||
size=VECTOR_DIMENSION,
|
||||
distance=Distance.Cosine,
|
||||
)
|
||||
@@ -29,7 +29,7 @@ config = EdgeConfig(
|
||||
# @block-start initialize-edge-shard
|
||||
from qdrant_edge import EdgeShard
|
||||
|
||||
edge_shard = EdgeShard(SHARD_DIRECTORY, config)
|
||||
edge_shard = EdgeShard.create(SHARD_DIRECTORY, config)
|
||||
# @block-end initialize-edge-shard
|
||||
|
||||
# @block-start upsert-points
|
||||
@@ -45,7 +45,7 @@ edge_shard.update(UpdateOperation.upsert_points([point]))
|
||||
# @block-end upsert-points
|
||||
|
||||
# @block-start retrieve-point
|
||||
point = edge_shard.retrieve(
|
||||
records = edge_shard.retrieve(
|
||||
point_ids=[1],
|
||||
with_payload=True,
|
||||
with_vector=False
|
||||
@@ -65,10 +65,65 @@ results = edge_shard.query(
|
||||
)
|
||||
# @block-end query-points
|
||||
|
||||
# @block-start filter
|
||||
from qdrant_edge import FieldCondition, Filter, MatchValue
|
||||
|
||||
results = edge_shard.query(
|
||||
QueryRequest(
|
||||
query=Query.Nearest([0.2, 0.1, 0.9, 0.7], using=VECTOR_NAME),
|
||||
filter=Filter(
|
||||
must=[
|
||||
FieldCondition(
|
||||
key="color",
|
||||
match=MatchValue(value="red"),
|
||||
)
|
||||
]
|
||||
),
|
||||
limit=10,
|
||||
with_vector=False,
|
||||
with_payload=True
|
||||
)
|
||||
)
|
||||
# @block-end filter
|
||||
|
||||
# @block-start facet
|
||||
from qdrant_edge import FacetRequest
|
||||
|
||||
facet_response = edge_shard.facet(FacetRequest(key="color", limit=10, exact=False))
|
||||
# @block-end facet
|
||||
|
||||
# @block-start optimize
|
||||
edge_shard.optimize()
|
||||
# @block-end optimize
|
||||
|
||||
# @block-start configure-optimizer
|
||||
from qdrant_edge import EdgeOptimizersConfig
|
||||
|
||||
config = EdgeConfig(
|
||||
vectors={
|
||||
VECTOR_NAME: EdgeVectorParams(
|
||||
size=VECTOR_DIMENSION,
|
||||
distance=Distance.Cosine,
|
||||
)
|
||||
},
|
||||
optimizers=EdgeOptimizersConfig(
|
||||
deleted_threshold=0.2,
|
||||
vacuum_min_vector_number=100,
|
||||
default_segment_number=2,
|
||||
),
|
||||
)
|
||||
# @block-end configure-optimizer
|
||||
|
||||
# @block-start create-payload-index
|
||||
from qdrant_edge import PayloadSchemaType
|
||||
|
||||
edge_shard.update(UpdateOperation.create_field_index("color", PayloadSchemaType.Keyword))
|
||||
# @block-end create-payload-index
|
||||
|
||||
# @block-start close-edge-shard
|
||||
edge_shard.close()
|
||||
# @block-end close-edge-shard
|
||||
|
||||
# @block-start load-edge-shard
|
||||
edge_shard = EdgeShard(SHARD_DIRECTORY)
|
||||
edge_shard = EdgeShard.load(SHARD_DIRECTORY)
|
||||
# @block-end load-edge-shard
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
|
||||
use qdrant_edge::EdgeShard;
|
||||
use qdrant_edge::{
|
||||
Condition, CreateIndex, Distance, EdgeConfig, EdgeOptimizersConfig,
|
||||
EdgeVectorParams, FacetRequest, FieldCondition, FieldIndexOperations,
|
||||
Filter, Match, MatchValue, NamedQuery, PayloadFieldSchema,
|
||||
PayloadSchemaType, PointId, PointInsertOperations, PointOperations,
|
||||
PointStruct, PointStructPersisted, QueryEnum, QueryRequest, ScoringQuery,
|
||||
UpdateOperation, ValueVariants, Vectors, WithPayloadInterface, WithVector,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
pub async fn main() -> anyhow::Result<()> {
|
||||
// @block-start create-storage-directory
|
||||
const SHARD_DIRECTORY: &str = "./qdrant-edge-directory";
|
||||
|
||||
fs_err::create_dir_all(SHARD_DIRECTORY)?;
|
||||
// @block-end create-storage-directory
|
||||
|
||||
// @block-start configure-edge-shard
|
||||
const VECTOR_NAME: &str = "my-vector";
|
||||
const VECTOR_DIMENSION: usize = 4;
|
||||
|
||||
let config = EdgeConfig {
|
||||
on_disk_payload: true,
|
||||
vectors: HashMap::from([(
|
||||
VECTOR_NAME.to_string(),
|
||||
EdgeVectorParams {
|
||||
size: VECTOR_DIMENSION,
|
||||
distance: Distance::Cosine,
|
||||
on_disk: Some(true),
|
||||
quantization_config: None,
|
||||
multivector_config: None,
|
||||
datatype: None,
|
||||
hnsw_config: None,
|
||||
},
|
||||
)]),
|
||||
sparse_vectors: HashMap::new(),
|
||||
hnsw_config: Default::default(),
|
||||
quantization_config: None,
|
||||
optimizers: Default::default(),
|
||||
};
|
||||
// @block-end configure-edge-shard
|
||||
|
||||
// @block-start initialize-edge-shard
|
||||
let edge_shard = EdgeShard::new(
|
||||
Path::new(SHARD_DIRECTORY),
|
||||
config,
|
||||
)?;
|
||||
// @block-end initialize-edge-shard
|
||||
|
||||
// @block-start upsert-points
|
||||
let points: Vec<PointStructPersisted> = vec![
|
||||
PointStruct::new(
|
||||
1u64,
|
||||
Vectors::new_named([(VECTOR_NAME, vec![0.1f32, 0.2, 0.3, 0.4])]),
|
||||
json!({"color": "red"}),
|
||||
)
|
||||
.into(),
|
||||
];
|
||||
|
||||
edge_shard.update(UpdateOperation::PointOperation(
|
||||
PointOperations::UpsertPoints(
|
||||
PointInsertOperations::PointsList(points),
|
||||
),
|
||||
))?;
|
||||
// @block-end upsert-points
|
||||
|
||||
// @block-start retrieve-point
|
||||
let retrieved = edge_shard.retrieve(
|
||||
&[PointId::NumId(1)],
|
||||
Some(WithPayloadInterface::Bool(true)),
|
||||
Some(WithVector::Bool(false)),
|
||||
)?;
|
||||
// @block-end retrieve-point
|
||||
|
||||
// @block-start query-points
|
||||
let results = edge_shard.query(QueryRequest {
|
||||
prefetches: vec![],
|
||||
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery {
|
||||
query: vec![0.2f32, 0.1, 0.9, 0.7].into(),
|
||||
using: Some(VECTOR_NAME.to_string()),
|
||||
}))),
|
||||
filter: None,
|
||||
score_threshold: None,
|
||||
limit: 10,
|
||||
offset: 0,
|
||||
params: None,
|
||||
with_vector: WithVector::Bool(false),
|
||||
with_payload: WithPayloadInterface::Bool(true),
|
||||
})?;
|
||||
// @block-end query-points
|
||||
|
||||
// @block-start filter
|
||||
let filter = Filter {
|
||||
should: None,
|
||||
min_should: None,
|
||||
must: Some(vec![Condition::Field(FieldCondition::new_match(
|
||||
"color".try_into().unwrap(),
|
||||
Match::Value(MatchValue {
|
||||
value: ValueVariants::String("red".to_string()),
|
||||
}),
|
||||
))]),
|
||||
must_not: None,
|
||||
};
|
||||
|
||||
let results = edge_shard.query(QueryRequest {
|
||||
prefetches: vec![],
|
||||
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery {
|
||||
query: vec![0.2f32, 0.1, 0.9, 0.7].into(),
|
||||
using: Some(VECTOR_NAME.to_string()),
|
||||
}))),
|
||||
filter: Some(filter),
|
||||
score_threshold: None,
|
||||
limit: 10,
|
||||
offset: 0,
|
||||
params: None,
|
||||
with_vector: WithVector::Bool(false),
|
||||
with_payload: WithPayloadInterface::Bool(true),
|
||||
})?;
|
||||
// @block-end filter
|
||||
|
||||
// @block-start facet
|
||||
let facet_response = edge_shard.facet(FacetRequest {
|
||||
key: "color".try_into().unwrap(),
|
||||
limit: 10,
|
||||
filter: None,
|
||||
exact: false,
|
||||
})?;
|
||||
// @block-end facet
|
||||
|
||||
// @block-start optimize
|
||||
edge_shard.optimize()?;
|
||||
// @block-end optimize
|
||||
|
||||
// @block-start configure-optimizer
|
||||
let config = EdgeConfig {
|
||||
on_disk_payload: true,
|
||||
vectors: HashMap::from([(
|
||||
VECTOR_NAME.to_string(),
|
||||
EdgeVectorParams {
|
||||
size: VECTOR_DIMENSION,
|
||||
distance: Distance::Cosine,
|
||||
on_disk: Some(true),
|
||||
quantization_config: None,
|
||||
multivector_config: None,
|
||||
datatype: None,
|
||||
hnsw_config: None,
|
||||
},
|
||||
)]),
|
||||
sparse_vectors: HashMap::new(),
|
||||
hnsw_config: Default::default(),
|
||||
quantization_config: None,
|
||||
optimizers: EdgeOptimizersConfig {
|
||||
deleted_threshold: Some(0.2),
|
||||
vacuum_min_vector_number: Some(100),
|
||||
default_segment_number: Some(2),
|
||||
..Default::default()
|
||||
},
|
||||
};
|
||||
// @block-end configure-optimizer
|
||||
|
||||
// @block-start create-payload-index
|
||||
edge_shard.update(UpdateOperation::FieldIndexOperation(
|
||||
FieldIndexOperations::CreateIndex(CreateIndex {
|
||||
field_name: "color".try_into().unwrap(),
|
||||
field_schema: Some(PayloadFieldSchema::FieldType(
|
||||
PayloadSchemaType::Keyword,
|
||||
)),
|
||||
}),
|
||||
))?;
|
||||
// @block-end create-payload-index
|
||||
|
||||
// @block-start close-edge-shard
|
||||
drop(edge_shard);
|
||||
// @block-end close-edge-shard
|
||||
|
||||
// @block-start load-edge-shard
|
||||
let edge_shard = EdgeShard::load(Path::new(SHARD_DIRECTORY), None)?;
|
||||
// @block-end load-edge-shard
|
||||
|
||||
Ok(())
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
```rust
|
||||
let filter = Filter::new_must(Condition::Field(FieldCondition::new_range(
|
||||
SYNC_TIMESTAMP_KEY.parse::<JsonPath>().unwrap(),
|
||||
Range {
|
||||
lte: Some(OrderedFloat(sync_timestamp)),
|
||||
..Default::default()
|
||||
},
|
||||
)));
|
||||
|
||||
mutable_shard.update(UpdateOperation::PointOperation(
|
||||
PointOperations::DeletePointsByFilter(filter),
|
||||
))?;
|
||||
```
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
```rust
|
||||
const SYNC_TIMESTAMP_KEY: &str = "timestamp";
|
||||
let id = 2u64;
|
||||
let vector = vec![0.4f32, 0.3, 0.2, 0.1];
|
||||
let timestamp = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs_f64();
|
||||
let payload = json!({
|
||||
"color": "green",
|
||||
SYNC_TIMESTAMP_KEY: timestamp,
|
||||
});
|
||||
|
||||
let edge_points: Vec<PointStructPersisted> = vec![
|
||||
EdgePoint::new(
|
||||
PointId::NumId(id),
|
||||
Vectors::new_named([(VECTOR_NAME, vector.clone())]),
|
||||
payload.clone(),
|
||||
)
|
||||
.into(),
|
||||
];
|
||||
mutable_shard.update(UpdateOperation::PointOperation(
|
||||
PointOperations::UpsertPoints(
|
||||
PointInsertOperations::PointsList(edge_points),
|
||||
),
|
||||
))?;
|
||||
|
||||
let rest_point = PointStruct::new(
|
||||
id,
|
||||
HashMap::from([(VECTOR_NAME.to_string(), vector)]),
|
||||
payload.as_object().cloned().unwrap_or_default(),
|
||||
);
|
||||
upload_queue.push_back(rest_point);
|
||||
```
|
||||
+1
-1
@@ -25,5 +25,5 @@ with tempfile.TemporaryDirectory(dir=data_dir.parent) as restore_dir:
|
||||
|
||||
EdgeShard.unpack_snapshot(str(snapshot_path), str(data_dir))
|
||||
|
||||
immutable_shard = EdgeShard(IMMUTABLE_SHARD_DIR)
|
||||
immutable_shard = EdgeShard.load(IMMUTABLE_SHARD_DIR)
|
||||
```
|
||||
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
```rust
|
||||
const COLLECTION_NAME: &str = "edge-collection";
|
||||
let snapshot_url = format!(
|
||||
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot"
|
||||
);
|
||||
|
||||
const IMMUTABLE_SHARD_DIR: &str = "./qdrant-edge-directory/immutable";
|
||||
let data_dir = Path::new(IMMUTABLE_SHARD_DIR);
|
||||
|
||||
let restore_dir = tempfile::Builder::new()
|
||||
.tempdir_in(data_dir.parent().unwrap_or(Path::new(".")))?;
|
||||
let snapshot_path = restore_dir.path().join("shard.snapshot");
|
||||
|
||||
let mut bytes = Vec::new();
|
||||
std::io::copy(
|
||||
&mut ureq::get(&snapshot_url)
|
||||
.header("api-key", QDRANT_API_KEY)
|
||||
.call()?
|
||||
.into_body()
|
||||
.into_reader(),
|
||||
&mut bytes,
|
||||
)?;
|
||||
fs_err::write(&snapshot_path, &bytes)?;
|
||||
|
||||
if data_dir.exists() {
|
||||
fs_err::remove_dir_all(data_dir)?;
|
||||
}
|
||||
fs_err::create_dir_all(data_dir)?;
|
||||
|
||||
EdgeShard::unpack_snapshot(&snapshot_path, data_dir)?;
|
||||
|
||||
let immutable_shard = EdgeShard::load(data_dir, None)?;
|
||||
```
|
||||
+4
-4
@@ -4,7 +4,7 @@ from qdrant_edge import (
|
||||
Distance,
|
||||
EdgeConfig,
|
||||
EdgeShard,
|
||||
VectorDataConfig,
|
||||
EdgeVectorParams,
|
||||
)
|
||||
|
||||
MUTABLE_SHARD_DIR = "./qdrant-edge-directory/mutable"
|
||||
@@ -15,13 +15,13 @@ VECTOR_NAME="my-vector"
|
||||
VECTOR_DIMENSION=4
|
||||
|
||||
config = EdgeConfig(
|
||||
vector_data={
|
||||
VECTOR_NAME: VectorDataConfig(
|
||||
vectors={
|
||||
VECTOR_NAME: EdgeVectorParams(
|
||||
size=VECTOR_DIMENSION,
|
||||
distance=Distance.Cosine,
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
mutable_shard = EdgeShard(MUTABLE_SHARD_DIR, config)
|
||||
mutable_shard = EdgeShard.create(MUTABLE_SHARD_DIR, config)
|
||||
```
|
||||
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
```rust
|
||||
const MUTABLE_SHARD_DIR: &str = "./qdrant-edge-directory/mutable";
|
||||
const VECTOR_DIMENSION: usize = 4;
|
||||
const VECTOR_NAME: &str = "my-vector";
|
||||
|
||||
fs_err::create_dir_all(MUTABLE_SHARD_DIR)?;
|
||||
let config = EdgeConfig {
|
||||
on_disk_payload: true,
|
||||
vectors: HashMap::from([(
|
||||
VECTOR_NAME.to_string(),
|
||||
EdgeVectorParams {
|
||||
size: VECTOR_DIMENSION,
|
||||
distance: Distance::Cosine,
|
||||
on_disk: Some(true),
|
||||
quantization_config: None,
|
||||
multivector_config: None,
|
||||
datatype: None,
|
||||
hnsw_config: None,
|
||||
},
|
||||
)]),
|
||||
sparse_vectors: HashMap::new(),
|
||||
hnsw_config: Default::default(),
|
||||
quantization_config: None,
|
||||
optimizers: Default::default(),
|
||||
};
|
||||
|
||||
let mutable_shard = EdgeShard::new(
|
||||
Path::new(MUTABLE_SHARD_DIR),
|
||||
config,
|
||||
)?;
|
||||
```
|
||||
+5
-5
@@ -4,7 +4,7 @@ from qdrant_edge import (
|
||||
Distance,
|
||||
EdgeConfig,
|
||||
EdgeShard,
|
||||
VectorDataConfig,
|
||||
EdgeVectorParams,
|
||||
)
|
||||
|
||||
MUTABLE_SHARD_DIR = "./qdrant-edge-directory/mutable"
|
||||
@@ -15,15 +15,15 @@ VECTOR_NAME="my-vector"
|
||||
VECTOR_DIMENSION=4
|
||||
|
||||
config = EdgeConfig(
|
||||
vector_data={
|
||||
VECTOR_NAME: VectorDataConfig(
|
||||
vectors={
|
||||
VECTOR_NAME: EdgeVectorParams(
|
||||
size=VECTOR_DIMENSION,
|
||||
distance=Distance.Cosine,
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
mutable_shard = EdgeShard(MUTABLE_SHARD_DIR, config)
|
||||
mutable_shard = EdgeShard.create(MUTABLE_SHARD_DIR, config)
|
||||
|
||||
import requests
|
||||
import tempfile
|
||||
@@ -51,7 +51,7 @@ with tempfile.TemporaryDirectory(dir=data_dir.parent) as restore_dir:
|
||||
|
||||
EdgeShard.unpack_snapshot(str(snapshot_path), str(data_dir))
|
||||
|
||||
immutable_shard = EdgeShard(IMMUTABLE_SHARD_DIR)
|
||||
immutable_shard = EdgeShard.load(IMMUTABLE_SHARD_DIR)
|
||||
|
||||
from qdrant_edge import ( Point, UpdateOperation )
|
||||
from qdrant_client import models
|
||||
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
```rust
|
||||
let query = QueryRequest {
|
||||
prefetches: vec![],
|
||||
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery {
|
||||
query: vec![0.2f32, 0.1, 0.9, 0.7].into(),
|
||||
using: Some(VECTOR_NAME.to_string()),
|
||||
}))),
|
||||
filter: None,
|
||||
score_threshold: None,
|
||||
limit: 10,
|
||||
offset: 0,
|
||||
params: None,
|
||||
with_vector: WithVector::Bool(false),
|
||||
with_payload: WithPayloadInterface::Bool(true),
|
||||
};
|
||||
|
||||
let mut all_results = mutable_shard.query(query.clone())?;
|
||||
all_results.extend(immutable_shard.query(query)?);
|
||||
|
||||
all_results.sort_by(|a, b| {
|
||||
b.score
|
||||
.partial_cmp(&a.score)
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
});
|
||||
|
||||
let mut seen_ids = std::collections::HashSet::new();
|
||||
let results: Vec<_> = all_results
|
||||
.into_iter()
|
||||
.filter(|p| seen_ids.insert(p.id.clone()))
|
||||
.take(10)
|
||||
.collect();
|
||||
```
|
||||
+196
@@ -0,0 +1,196 @@
|
||||
```rust
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use ordered_float::OrderedFloat;
|
||||
use qdrant_client::qdrant::PointStruct;
|
||||
use qdrant_edge::EdgeShard;
|
||||
use qdrant_edge::internal::SnapshotManifest;
|
||||
use qdrant_edge::{
|
||||
Condition, Distance, EdgeConfig, EdgeVectorParams, FieldCondition, Filter,
|
||||
JsonPath, NamedQuery, PointId, PointInsertOperations, PointOperations,
|
||||
PointStruct as EdgePoint, PointStructPersisted, QueryEnum, QueryRequest,
|
||||
Range, ScoringQuery, UpdateOperation, Vectors, WithPayloadInterface,
|
||||
WithVector,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
const MUTABLE_SHARD_DIR: &str = "./qdrant-edge-directory/mutable";
|
||||
const VECTOR_DIMENSION: usize = 4;
|
||||
const VECTOR_NAME: &str = "my-vector";
|
||||
|
||||
fs_err::create_dir_all(MUTABLE_SHARD_DIR)?;
|
||||
let config = EdgeConfig {
|
||||
on_disk_payload: true,
|
||||
vectors: HashMap::from([(
|
||||
VECTOR_NAME.to_string(),
|
||||
EdgeVectorParams {
|
||||
size: VECTOR_DIMENSION,
|
||||
distance: Distance::Cosine,
|
||||
on_disk: Some(true),
|
||||
quantization_config: None,
|
||||
multivector_config: None,
|
||||
datatype: None,
|
||||
hnsw_config: None,
|
||||
},
|
||||
)]),
|
||||
sparse_vectors: HashMap::new(),
|
||||
hnsw_config: Default::default(),
|
||||
quantization_config: None,
|
||||
optimizers: Default::default(),
|
||||
};
|
||||
|
||||
let mutable_shard = EdgeShard::new(
|
||||
Path::new(MUTABLE_SHARD_DIR),
|
||||
config,
|
||||
)?;
|
||||
|
||||
const COLLECTION_NAME: &str = "edge-collection";
|
||||
let snapshot_url = format!(
|
||||
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot"
|
||||
);
|
||||
|
||||
const IMMUTABLE_SHARD_DIR: &str = "./qdrant-edge-directory/immutable";
|
||||
let data_dir = Path::new(IMMUTABLE_SHARD_DIR);
|
||||
|
||||
let restore_dir = tempfile::Builder::new()
|
||||
.tempdir_in(data_dir.parent().unwrap_or(Path::new(".")))?;
|
||||
let snapshot_path = restore_dir.path().join("shard.snapshot");
|
||||
|
||||
let mut bytes = Vec::new();
|
||||
std::io::copy(
|
||||
&mut ureq::get(&snapshot_url)
|
||||
.header("api-key", QDRANT_API_KEY)
|
||||
.call()?
|
||||
.into_body()
|
||||
.into_reader(),
|
||||
&mut bytes,
|
||||
)?;
|
||||
fs_err::write(&snapshot_path, &bytes)?;
|
||||
|
||||
if data_dir.exists() {
|
||||
fs_err::remove_dir_all(data_dir)?;
|
||||
}
|
||||
fs_err::create_dir_all(data_dir)?;
|
||||
|
||||
EdgeShard::unpack_snapshot(&snapshot_path, data_dir)?;
|
||||
|
||||
let immutable_shard = EdgeShard::load(data_dir, None)?;
|
||||
|
||||
const SYNC_TIMESTAMP_KEY: &str = "timestamp";
|
||||
let id = 2u64;
|
||||
let vector = vec![0.4f32, 0.3, 0.2, 0.1];
|
||||
let timestamp = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs_f64();
|
||||
let payload = json!({
|
||||
"color": "green",
|
||||
SYNC_TIMESTAMP_KEY: timestamp,
|
||||
});
|
||||
|
||||
let edge_points: Vec<PointStructPersisted> = vec![
|
||||
EdgePoint::new(
|
||||
PointId::NumId(id),
|
||||
Vectors::new_named([(VECTOR_NAME, vector.clone())]),
|
||||
payload.clone(),
|
||||
)
|
||||
.into(),
|
||||
];
|
||||
mutable_shard.update(UpdateOperation::PointOperation(
|
||||
PointOperations::UpsertPoints(
|
||||
PointInsertOperations::PointsList(edge_points),
|
||||
),
|
||||
))?;
|
||||
|
||||
let rest_point = PointStruct::new(
|
||||
id,
|
||||
HashMap::from([(VECTOR_NAME.to_string(), vector)]),
|
||||
payload.as_object().cloned().unwrap_or_default(),
|
||||
);
|
||||
upload_queue.push_back(rest_point);
|
||||
|
||||
let sync_timestamp = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs_f64();
|
||||
|
||||
let current_manifest = immutable_shard.snapshot_manifest()?;
|
||||
|
||||
let update_url = format!(
|
||||
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot\
|
||||
/partial/create"
|
||||
);
|
||||
|
||||
let temp_dir = tempfile::tempdir_in(data_dir)?;
|
||||
let partial_snapshot_path = temp_dir.path().join("partial.snapshot");
|
||||
|
||||
let mut bytes = Vec::new();
|
||||
std::io::copy(
|
||||
&mut ureq::post(&update_url)
|
||||
.header("api-key", QDRANT_API_KEY)
|
||||
.send_json(¤t_manifest)?
|
||||
.into_body()
|
||||
.into_reader(),
|
||||
&mut bytes,
|
||||
)?;
|
||||
fs_err::write(&partial_snapshot_path, &bytes)?;
|
||||
|
||||
let unpacked_dir = tempfile::tempdir_in(data_dir)?;
|
||||
EdgeShard::unpack_snapshot(&partial_snapshot_path, unpacked_dir.path())?;
|
||||
let snapshot_manifest = SnapshotManifest::load_from_snapshot(
|
||||
unpacked_dir.path(),
|
||||
None,
|
||||
)?;
|
||||
|
||||
let immutable_shard = EdgeShard::recover_partial_snapshot(
|
||||
data_dir,
|
||||
¤t_manifest,
|
||||
unpacked_dir.path(),
|
||||
&snapshot_manifest,
|
||||
)?;
|
||||
|
||||
let filter = Filter::new_must(Condition::Field(FieldCondition::new_range(
|
||||
SYNC_TIMESTAMP_KEY.parse::<JsonPath>().unwrap(),
|
||||
Range {
|
||||
lte: Some(OrderedFloat(sync_timestamp)),
|
||||
..Default::default()
|
||||
},
|
||||
)));
|
||||
|
||||
mutable_shard.update(UpdateOperation::PointOperation(
|
||||
PointOperations::DeletePointsByFilter(filter),
|
||||
))?;
|
||||
|
||||
let query = QueryRequest {
|
||||
prefetches: vec![],
|
||||
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery {
|
||||
query: vec![0.2f32, 0.1, 0.9, 0.7].into(),
|
||||
using: Some(VECTOR_NAME.to_string()),
|
||||
}))),
|
||||
filter: None,
|
||||
score_threshold: None,
|
||||
limit: 10,
|
||||
offset: 0,
|
||||
params: None,
|
||||
with_vector: WithVector::Bool(false),
|
||||
with_payload: WithPayloadInterface::Bool(true),
|
||||
};
|
||||
|
||||
let mut all_results = mutable_shard.query(query.clone())?;
|
||||
all_results.extend(immutable_shard.query(query)?);
|
||||
|
||||
all_results.sort_by(|a, b| {
|
||||
b.score
|
||||
.partial_cmp(&a.score)
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
});
|
||||
|
||||
let mut seen_ids = std::collections::HashSet::new();
|
||||
let results: Vec<_> = all_results
|
||||
.into_iter()
|
||||
.filter(|p| seen_ids.insert(p.id.clone()))
|
||||
.take(10)
|
||||
.collect();
|
||||
```
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
```rust
|
||||
let sync_timestamp = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs_f64();
|
||||
|
||||
let current_manifest = immutable_shard.snapshot_manifest()?;
|
||||
|
||||
let update_url = format!(
|
||||
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot\
|
||||
/partial/create"
|
||||
);
|
||||
|
||||
let temp_dir = tempfile::tempdir_in(data_dir)?;
|
||||
let partial_snapshot_path = temp_dir.path().join("partial.snapshot");
|
||||
|
||||
let mut bytes = Vec::new();
|
||||
std::io::copy(
|
||||
&mut ureq::post(&update_url)
|
||||
.header("api-key", QDRANT_API_KEY)
|
||||
.send_json(¤t_manifest)?
|
||||
.into_body()
|
||||
.into_reader(),
|
||||
&mut bytes,
|
||||
)?;
|
||||
fs_err::write(&partial_snapshot_path, &bytes)?;
|
||||
|
||||
let unpacked_dir = tempfile::tempdir_in(data_dir)?;
|
||||
EdgeShard::unpack_snapshot(&partial_snapshot_path, unpacked_dir.path())?;
|
||||
let snapshot_manifest = SnapshotManifest::load_from_snapshot(
|
||||
unpacked_dir.path(),
|
||||
None,
|
||||
)?;
|
||||
|
||||
let immutable_shard = EdgeShard::recover_partial_snapshot(
|
||||
data_dir,
|
||||
¤t_manifest,
|
||||
unpacked_dir.path(),
|
||||
&snapshot_manifest,
|
||||
)?;
|
||||
```
|
||||
+5
-5
@@ -13,7 +13,7 @@ from qdrant_edge import (
|
||||
Distance,
|
||||
EdgeConfig,
|
||||
EdgeShard,
|
||||
VectorDataConfig,
|
||||
EdgeVectorParams,
|
||||
)
|
||||
|
||||
MUTABLE_SHARD_DIR = "./qdrant-edge-directory/mutable"
|
||||
@@ -24,15 +24,15 @@ VECTOR_NAME="my-vector"
|
||||
VECTOR_DIMENSION=4
|
||||
|
||||
config = EdgeConfig(
|
||||
vector_data={
|
||||
VECTOR_NAME: VectorDataConfig(
|
||||
vectors={
|
||||
VECTOR_NAME: EdgeVectorParams(
|
||||
size=VECTOR_DIMENSION,
|
||||
distance=Distance.Cosine,
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
mutable_shard = EdgeShard(MUTABLE_SHARD_DIR, config)
|
||||
mutable_shard = EdgeShard.create(MUTABLE_SHARD_DIR, config)
|
||||
# @block-end initialize-mutable-shard
|
||||
|
||||
# @block-start initialize-immutable-shard
|
||||
@@ -62,7 +62,7 @@ with tempfile.TemporaryDirectory(dir=data_dir.parent) as restore_dir:
|
||||
|
||||
EdgeShard.unpack_snapshot(str(snapshot_path), str(data_dir))
|
||||
|
||||
immutable_shard = EdgeShard(IMMUTABLE_SHARD_DIR)
|
||||
immutable_shard = EdgeShard.load(IMMUTABLE_SHARD_DIR)
|
||||
# @block-end initialize-immutable-shard
|
||||
|
||||
# @block-start dual-write
|
||||
|
||||
+217
@@ -0,0 +1,217 @@
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use ordered_float::OrderedFloat;
|
||||
use qdrant_client::qdrant::PointStruct;
|
||||
use qdrant_edge::EdgeShard;
|
||||
use qdrant_edge::internal::SnapshotManifest;
|
||||
use qdrant_edge::{
|
||||
Condition, Distance, EdgeConfig, EdgeVectorParams, FieldCondition, Filter,
|
||||
JsonPath, NamedQuery, PointId, PointInsertOperations, PointOperations,
|
||||
PointStruct as EdgePoint, PointStructPersisted, QueryEnum, QueryRequest,
|
||||
Range, ScoringQuery, UpdateOperation, Vectors, WithPayloadInterface,
|
||||
WithVector,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
pub async fn main() -> anyhow::Result<()> {
|
||||
// @hide-start
|
||||
const QDRANT_URL: &str = "";
|
||||
const QDRANT_API_KEY: &str = "";
|
||||
let mut upload_queue: std::collections::VecDeque<PointStruct> =
|
||||
std::collections::VecDeque::new();
|
||||
// @hide-end
|
||||
|
||||
// @block-start initialize-mutable-shard
|
||||
const MUTABLE_SHARD_DIR: &str = "./qdrant-edge-directory/mutable";
|
||||
const VECTOR_DIMENSION: usize = 4;
|
||||
const VECTOR_NAME: &str = "my-vector";
|
||||
|
||||
fs_err::create_dir_all(MUTABLE_SHARD_DIR)?;
|
||||
let config = EdgeConfig {
|
||||
on_disk_payload: true,
|
||||
vectors: HashMap::from([(
|
||||
VECTOR_NAME.to_string(),
|
||||
EdgeVectorParams {
|
||||
size: VECTOR_DIMENSION,
|
||||
distance: Distance::Cosine,
|
||||
on_disk: Some(true),
|
||||
quantization_config: None,
|
||||
multivector_config: None,
|
||||
datatype: None,
|
||||
hnsw_config: None,
|
||||
},
|
||||
)]),
|
||||
sparse_vectors: HashMap::new(),
|
||||
hnsw_config: Default::default(),
|
||||
quantization_config: None,
|
||||
optimizers: Default::default(),
|
||||
};
|
||||
|
||||
let mutable_shard = EdgeShard::new(
|
||||
Path::new(MUTABLE_SHARD_DIR),
|
||||
config,
|
||||
)?;
|
||||
// @block-end initialize-mutable-shard
|
||||
|
||||
// @block-start initialize-immutable-shard
|
||||
const COLLECTION_NAME: &str = "edge-collection";
|
||||
let snapshot_url = format!(
|
||||
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot"
|
||||
);
|
||||
|
||||
const IMMUTABLE_SHARD_DIR: &str = "./qdrant-edge-directory/immutable";
|
||||
let data_dir = Path::new(IMMUTABLE_SHARD_DIR);
|
||||
|
||||
let restore_dir = tempfile::Builder::new()
|
||||
.tempdir_in(data_dir.parent().unwrap_or(Path::new(".")))?;
|
||||
let snapshot_path = restore_dir.path().join("shard.snapshot");
|
||||
|
||||
let mut bytes = Vec::new();
|
||||
std::io::copy(
|
||||
&mut ureq::get(&snapshot_url)
|
||||
.header("api-key", QDRANT_API_KEY)
|
||||
.call()?
|
||||
.into_body()
|
||||
.into_reader(),
|
||||
&mut bytes,
|
||||
)?;
|
||||
fs_err::write(&snapshot_path, &bytes)?;
|
||||
|
||||
if data_dir.exists() {
|
||||
fs_err::remove_dir_all(data_dir)?;
|
||||
}
|
||||
fs_err::create_dir_all(data_dir)?;
|
||||
|
||||
EdgeShard::unpack_snapshot(&snapshot_path, data_dir)?;
|
||||
|
||||
let immutable_shard = EdgeShard::load(data_dir, None)?;
|
||||
// @block-end initialize-immutable-shard
|
||||
|
||||
// @block-start dual-write
|
||||
const SYNC_TIMESTAMP_KEY: &str = "timestamp";
|
||||
let id = 2u64;
|
||||
let vector = vec![0.4f32, 0.3, 0.2, 0.1];
|
||||
let timestamp = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs_f64();
|
||||
let payload = json!({
|
||||
"color": "green",
|
||||
SYNC_TIMESTAMP_KEY: timestamp,
|
||||
});
|
||||
|
||||
let edge_points: Vec<PointStructPersisted> = vec![
|
||||
EdgePoint::new(
|
||||
PointId::NumId(id),
|
||||
Vectors::new_named([(VECTOR_NAME, vector.clone())]),
|
||||
payload.clone(),
|
||||
)
|
||||
.into(),
|
||||
];
|
||||
mutable_shard.update(UpdateOperation::PointOperation(
|
||||
PointOperations::UpsertPoints(
|
||||
PointInsertOperations::PointsList(edge_points),
|
||||
),
|
||||
))?;
|
||||
|
||||
let rest_point = PointStruct::new(
|
||||
id,
|
||||
HashMap::from([(VECTOR_NAME.to_string(), vector)]),
|
||||
payload.as_object().cloned().unwrap_or_default(),
|
||||
);
|
||||
upload_queue.push_back(rest_point);
|
||||
// @block-end dual-write
|
||||
|
||||
// @block-start update-immutable-shard
|
||||
let sync_timestamp = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs_f64();
|
||||
|
||||
let current_manifest = immutable_shard.snapshot_manifest()?;
|
||||
|
||||
let update_url = format!(
|
||||
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot\
|
||||
/partial/create"
|
||||
);
|
||||
|
||||
let temp_dir = tempfile::tempdir_in(data_dir)?;
|
||||
let partial_snapshot_path = temp_dir.path().join("partial.snapshot");
|
||||
|
||||
let mut bytes = Vec::new();
|
||||
std::io::copy(
|
||||
&mut ureq::post(&update_url)
|
||||
.header("api-key", QDRANT_API_KEY)
|
||||
.send_json(¤t_manifest)?
|
||||
.into_body()
|
||||
.into_reader(),
|
||||
&mut bytes,
|
||||
)?;
|
||||
fs_err::write(&partial_snapshot_path, &bytes)?;
|
||||
|
||||
let unpacked_dir = tempfile::tempdir_in(data_dir)?;
|
||||
EdgeShard::unpack_snapshot(&partial_snapshot_path, unpacked_dir.path())?;
|
||||
let snapshot_manifest = SnapshotManifest::load_from_snapshot(
|
||||
unpacked_dir.path(),
|
||||
None,
|
||||
)?;
|
||||
|
||||
let immutable_shard = EdgeShard::recover_partial_snapshot(
|
||||
data_dir,
|
||||
¤t_manifest,
|
||||
unpacked_dir.path(),
|
||||
&snapshot_manifest,
|
||||
)?;
|
||||
// @block-end update-immutable-shard
|
||||
|
||||
// @block-start delete-synced-points
|
||||
let filter = Filter::new_must(Condition::Field(FieldCondition::new_range(
|
||||
SYNC_TIMESTAMP_KEY.parse::<JsonPath>().unwrap(),
|
||||
Range {
|
||||
lte: Some(OrderedFloat(sync_timestamp)),
|
||||
..Default::default()
|
||||
},
|
||||
)));
|
||||
|
||||
mutable_shard.update(UpdateOperation::PointOperation(
|
||||
PointOperations::DeletePointsByFilter(filter),
|
||||
))?;
|
||||
// @block-end delete-synced-points
|
||||
|
||||
// @block-start query-both-shards
|
||||
let query = QueryRequest {
|
||||
prefetches: vec![],
|
||||
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery {
|
||||
query: vec![0.2f32, 0.1, 0.9, 0.7].into(),
|
||||
using: Some(VECTOR_NAME.to_string()),
|
||||
}))),
|
||||
filter: None,
|
||||
score_threshold: None,
|
||||
limit: 10,
|
||||
offset: 0,
|
||||
params: None,
|
||||
with_vector: WithVector::Bool(false),
|
||||
with_payload: WithPayloadInterface::Bool(true),
|
||||
};
|
||||
|
||||
let mut all_results = mutable_shard.query(query.clone())?;
|
||||
all_results.extend(immutable_shard.query(query)?);
|
||||
|
||||
all_results.sort_by(|a, b| {
|
||||
b.score
|
||||
.partial_cmp(&a.score)
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
});
|
||||
|
||||
let mut seen_ids = std::collections::HashSet::new();
|
||||
let results: Vec<_> = all_results
|
||||
.into_iter()
|
||||
.filter(|p| seen_ids.insert(p.id.clone()))
|
||||
.take(10)
|
||||
.collect();
|
||||
// @block-end query-both-shards
|
||||
|
||||
Ok(())
|
||||
}
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
```rust
|
||||
// This is an in-memory queue.
|
||||
// For production use cases consider persisting changes.
|
||||
let mut upload_queue: std::collections::VecDeque<PointStruct> =
|
||||
std::collections::VecDeque::new();
|
||||
```
|
||||
+4
-4
@@ -3,7 +3,7 @@ from pathlib import Path
|
||||
from qdrant_edge import (
|
||||
Distance,
|
||||
EdgeConfig,
|
||||
VectorDataConfig,
|
||||
EdgeVectorParams,
|
||||
)
|
||||
|
||||
SHARD_DIRECTORY = "./qdrant-edge-directory"
|
||||
@@ -12,13 +12,13 @@ VECTOR_DIMENSION=4
|
||||
|
||||
Path(SHARD_DIRECTORY).mkdir(parents=True, exist_ok=True)
|
||||
config = EdgeConfig(
|
||||
vector_data={
|
||||
VECTOR_NAME: VectorDataConfig(
|
||||
vectors={
|
||||
VECTOR_NAME: EdgeVectorParams(
|
||||
size=VECTOR_DIMENSION,
|
||||
distance=Distance.Cosine,
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
edge_shard = EdgeShard(SHARD_DIRECTORY, config)
|
||||
edge_shard = EdgeShard.create(SHARD_DIRECTORY, config)
|
||||
```
|
||||
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
```rust
|
||||
const VECTOR_DIMENSION: usize = 4;
|
||||
const VECTOR_NAME: &str = "my-vector";
|
||||
|
||||
fs_err::create_dir_all(SHARD_DIRECTORY)?;
|
||||
let config = EdgeConfig {
|
||||
on_disk_payload: true,
|
||||
vectors: HashMap::from([(
|
||||
VECTOR_NAME.to_string(),
|
||||
EdgeVectorParams {
|
||||
size: VECTOR_DIMENSION,
|
||||
distance: qdrant_edge::Distance::Cosine,
|
||||
on_disk: Some(true),
|
||||
quantization_config: None,
|
||||
multivector_config: None,
|
||||
datatype: None,
|
||||
hnsw_config: None,
|
||||
},
|
||||
)]),
|
||||
sparse_vectors: HashMap::new(),
|
||||
hnsw_config: Default::default(),
|
||||
quantization_config: None,
|
||||
optimizers: Default::default(),
|
||||
};
|
||||
|
||||
let edge_shard = EdgeShard::new(
|
||||
Path::new(SHARD_DIRECTORY),
|
||||
config,
|
||||
)?;
|
||||
```
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
```rust
|
||||
let server_client = Qdrant::from_url(QDRANT_URL)
|
||||
.api_key(QDRANT_API_KEY)
|
||||
.build()?;
|
||||
|
||||
if !server_client.collection_exists(COLLECTION_NAME).await? {
|
||||
server_client
|
||||
.create_collection(
|
||||
CreateCollectionBuilder::new(COLLECTION_NAME).vectors_config(
|
||||
VectorParamsBuilder::new(
|
||||
VECTOR_DIMENSION as u64,
|
||||
Distance::Cosine,
|
||||
),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
```
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
```rust
|
||||
const BATCH_SIZE: usize = 10;
|
||||
let points_to_upload: Vec<PointStruct> = upload_queue
|
||||
.drain(..BATCH_SIZE.min(upload_queue.len()))
|
||||
.collect();
|
||||
|
||||
if !points_to_upload.is_empty() {
|
||||
server_client
|
||||
.upsert_points(UpsertPointsBuilder::new(
|
||||
COLLECTION_NAME,
|
||||
points_to_upload,
|
||||
))
|
||||
.await?;
|
||||
}
|
||||
```
|
||||
+5
-5
@@ -27,7 +27,7 @@ with tempfile.TemporaryDirectory(dir=data_dir.parent) as restore_dir:
|
||||
|
||||
EdgeShard.unpack_snapshot(str(snapshot_path), str(data_dir))
|
||||
|
||||
edge_shard = EdgeShard(SHARD_DIRECTORY)
|
||||
edge_shard = EdgeShard.load(SHARD_DIRECTORY)
|
||||
|
||||
manifest = edge_shard.snapshot_manifest()
|
||||
|
||||
@@ -48,7 +48,7 @@ from pathlib import Path
|
||||
from qdrant_edge import (
|
||||
Distance,
|
||||
EdgeConfig,
|
||||
VectorDataConfig,
|
||||
EdgeVectorParams,
|
||||
)
|
||||
|
||||
SHARD_DIRECTORY = "./qdrant-edge-directory"
|
||||
@@ -57,15 +57,15 @@ VECTOR_DIMENSION=4
|
||||
|
||||
Path(SHARD_DIRECTORY).mkdir(parents=True, exist_ok=True)
|
||||
config = EdgeConfig(
|
||||
vector_data={
|
||||
VECTOR_NAME: VectorDataConfig(
|
||||
vectors={
|
||||
VECTOR_NAME: EdgeVectorParams(
|
||||
size=VECTOR_DIMENSION,
|
||||
distance=Distance.Cosine,
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
edge_shard = EdgeShard(SHARD_DIRECTORY, config)
|
||||
edge_shard = EdgeShard.create(SHARD_DIRECTORY, config)
|
||||
|
||||
from qdrant_client import QdrantClient, models
|
||||
|
||||
|
||||
+1
-1
@@ -23,5 +23,5 @@ with tempfile.TemporaryDirectory(dir=data_dir.parent) as restore_dir:
|
||||
|
||||
EdgeShard.unpack_snapshot(str(snapshot_path), str(data_dir))
|
||||
|
||||
edge_shard = EdgeShard(SHARD_DIRECTORY)
|
||||
edge_shard = EdgeShard.load(SHARD_DIRECTORY)
|
||||
```
|
||||
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
```rust
|
||||
const SHARD_DIRECTORY: &str = "./qdrant-edge-directory";
|
||||
let data_dir = Path::new(SHARD_DIRECTORY);
|
||||
|
||||
let restore_dir = tempfile::Builder::new()
|
||||
.tempdir_in(data_dir.parent().unwrap_or(Path::new(".")))?;
|
||||
let snapshot_path = restore_dir.path().join("shard.snapshot");
|
||||
|
||||
let mut bytes = Vec::new();
|
||||
std::io::copy(
|
||||
&mut ureq::get(&snapshot_url)
|
||||
.header("api-key", QDRANT_API_KEY)
|
||||
.call()?
|
||||
.into_body()
|
||||
.into_reader(),
|
||||
&mut bytes,
|
||||
)?;
|
||||
fs_err::write(&snapshot_path, &bytes)?;
|
||||
|
||||
if data_dir.exists() {
|
||||
fs_err::remove_dir_all(data_dir)?;
|
||||
}
|
||||
fs_err::create_dir_all(data_dir)?;
|
||||
|
||||
EdgeShard::unpack_snapshot(&snapshot_path, data_dir)?;
|
||||
|
||||
let edge_shard = EdgeShard::load(data_dir, None)?;
|
||||
```
|
||||
+176
@@ -0,0 +1,176 @@
|
||||
```rust
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
|
||||
use qdrant_client::qdrant::{
|
||||
CreateCollectionBuilder, Distance, PointStruct, UpsertPointsBuilder,
|
||||
VectorParamsBuilder,
|
||||
};
|
||||
use qdrant_client::Qdrant;
|
||||
use qdrant_edge::EdgeShard;
|
||||
use qdrant_edge::internal::SnapshotManifest;
|
||||
use qdrant_edge::{
|
||||
EdgeConfig, EdgeVectorParams, PointId, PointInsertOperations,
|
||||
PointOperations, PointStruct as EdgePoint, PointStructPersisted,
|
||||
UpdateOperation, Vectors,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
const COLLECTION_NAME: &str = "edge-collection";
|
||||
|
||||
let snapshot_url = format!(
|
||||
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot"
|
||||
);
|
||||
|
||||
const SHARD_DIRECTORY: &str = "./qdrant-edge-directory";
|
||||
let data_dir = Path::new(SHARD_DIRECTORY);
|
||||
|
||||
let restore_dir = tempfile::Builder::new()
|
||||
.tempdir_in(data_dir.parent().unwrap_or(Path::new(".")))?;
|
||||
let snapshot_path = restore_dir.path().join("shard.snapshot");
|
||||
|
||||
let mut bytes = Vec::new();
|
||||
std::io::copy(
|
||||
&mut ureq::get(&snapshot_url)
|
||||
.header("api-key", QDRANT_API_KEY)
|
||||
.call()?
|
||||
.into_body()
|
||||
.into_reader(),
|
||||
&mut bytes,
|
||||
)?;
|
||||
fs_err::write(&snapshot_path, &bytes)?;
|
||||
|
||||
if data_dir.exists() {
|
||||
fs_err::remove_dir_all(data_dir)?;
|
||||
}
|
||||
fs_err::create_dir_all(data_dir)?;
|
||||
|
||||
EdgeShard::unpack_snapshot(&snapshot_path, data_dir)?;
|
||||
|
||||
let edge_shard = EdgeShard::load(data_dir, None)?;
|
||||
|
||||
let current_manifest = edge_shard.snapshot_manifest()?;
|
||||
|
||||
let update_url = format!(
|
||||
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot\
|
||||
/partial/create"
|
||||
);
|
||||
|
||||
let temp_dir = tempfile::tempdir_in(data_dir)?;
|
||||
let partial_snapshot_path = temp_dir.path().join("partial.snapshot");
|
||||
|
||||
let mut bytes = Vec::new();
|
||||
std::io::copy(
|
||||
&mut ureq::post(&update_url)
|
||||
.header("api-key", QDRANT_API_KEY)
|
||||
.send_json(¤t_manifest)?
|
||||
.into_body()
|
||||
.into_reader(),
|
||||
&mut bytes,
|
||||
)?;
|
||||
fs_err::write(&partial_snapshot_path, &bytes)?;
|
||||
|
||||
let unpacked_dir = tempfile::tempdir_in(data_dir)?;
|
||||
EdgeShard::unpack_snapshot(&partial_snapshot_path, unpacked_dir.path())?;
|
||||
let snapshot_manifest = SnapshotManifest::load_from_snapshot(
|
||||
unpacked_dir.path(),
|
||||
None,
|
||||
)?;
|
||||
|
||||
let edge_shard = EdgeShard::recover_partial_snapshot(
|
||||
data_dir,
|
||||
¤t_manifest,
|
||||
unpacked_dir.path(),
|
||||
&snapshot_manifest,
|
||||
)?;
|
||||
|
||||
const VECTOR_DIMENSION: usize = 4;
|
||||
const VECTOR_NAME: &str = "my-vector";
|
||||
|
||||
fs_err::create_dir_all(SHARD_DIRECTORY)?;
|
||||
let config = EdgeConfig {
|
||||
on_disk_payload: true,
|
||||
vectors: HashMap::from([(
|
||||
VECTOR_NAME.to_string(),
|
||||
EdgeVectorParams {
|
||||
size: VECTOR_DIMENSION,
|
||||
distance: qdrant_edge::Distance::Cosine,
|
||||
on_disk: Some(true),
|
||||
quantization_config: None,
|
||||
multivector_config: None,
|
||||
datatype: None,
|
||||
hnsw_config: None,
|
||||
},
|
||||
)]),
|
||||
sparse_vectors: HashMap::new(),
|
||||
hnsw_config: Default::default(),
|
||||
quantization_config: None,
|
||||
optimizers: Default::default(),
|
||||
};
|
||||
|
||||
let edge_shard = EdgeShard::new(
|
||||
Path::new(SHARD_DIRECTORY),
|
||||
config,
|
||||
)?;
|
||||
|
||||
let server_client = Qdrant::from_url(QDRANT_URL)
|
||||
.api_key(QDRANT_API_KEY)
|
||||
.build()?;
|
||||
|
||||
if !server_client.collection_exists(COLLECTION_NAME).await? {
|
||||
server_client
|
||||
.create_collection(
|
||||
CreateCollectionBuilder::new(COLLECTION_NAME).vectors_config(
|
||||
VectorParamsBuilder::new(
|
||||
VECTOR_DIMENSION as u64,
|
||||
Distance::Cosine,
|
||||
),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
// This is an in-memory queue.
|
||||
// For production use cases consider persisting changes.
|
||||
let mut upload_queue: std::collections::VecDeque<PointStruct> =
|
||||
std::collections::VecDeque::new();
|
||||
|
||||
let id = 1u64;
|
||||
let vector = vec![0.1f32, 0.2, 0.3, 0.4];
|
||||
let payload = json!({"color": "red"});
|
||||
|
||||
let edge_points: Vec<PointStructPersisted> = vec![
|
||||
EdgePoint::new(
|
||||
PointId::NumId(id),
|
||||
Vectors::new_named([(VECTOR_NAME, vector.clone())]),
|
||||
payload.clone(),
|
||||
)
|
||||
.into(),
|
||||
];
|
||||
edge_shard.update(UpdateOperation::PointOperation(
|
||||
PointOperations::UpsertPoints(
|
||||
PointInsertOperations::PointsList(edge_points),
|
||||
),
|
||||
))?;
|
||||
|
||||
let server_point = PointStruct::new(
|
||||
id,
|
||||
HashMap::from([(VECTOR_NAME.to_string(), vector)]),
|
||||
payload.as_object().cloned().unwrap_or_default(),
|
||||
);
|
||||
upload_queue.push_back(server_point);
|
||||
|
||||
const BATCH_SIZE: usize = 10;
|
||||
let points_to_upload: Vec<PointStruct> = upload_queue
|
||||
.drain(..BATCH_SIZE.min(upload_queue.len()))
|
||||
.collect();
|
||||
|
||||
if !points_to_upload.is_empty() {
|
||||
server_client
|
||||
.upsert_points(UpsertPointsBuilder::new(
|
||||
COLLECTION_NAME,
|
||||
points_to_upload,
|
||||
))
|
||||
.await?;
|
||||
}
|
||||
```
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
```rust
|
||||
const COLLECTION_NAME: &str = "edge-collection";
|
||||
|
||||
let snapshot_url = format!(
|
||||
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot"
|
||||
);
|
||||
```
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
```rust
|
||||
let current_manifest = edge_shard.snapshot_manifest()?;
|
||||
|
||||
let update_url = format!(
|
||||
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot\
|
||||
/partial/create"
|
||||
);
|
||||
|
||||
let temp_dir = tempfile::tempdir_in(data_dir)?;
|
||||
let partial_snapshot_path = temp_dir.path().join("partial.snapshot");
|
||||
|
||||
let mut bytes = Vec::new();
|
||||
std::io::copy(
|
||||
&mut ureq::post(&update_url)
|
||||
.header("api-key", QDRANT_API_KEY)
|
||||
.send_json(¤t_manifest)?
|
||||
.into_body()
|
||||
.into_reader(),
|
||||
&mut bytes,
|
||||
)?;
|
||||
fs_err::write(&partial_snapshot_path, &bytes)?;
|
||||
|
||||
let unpacked_dir = tempfile::tempdir_in(data_dir)?;
|
||||
EdgeShard::unpack_snapshot(&partial_snapshot_path, unpacked_dir.path())?;
|
||||
let snapshot_manifest = SnapshotManifest::load_from_snapshot(
|
||||
unpacked_dir.path(),
|
||||
None,
|
||||
)?;
|
||||
|
||||
let edge_shard = EdgeShard::recover_partial_snapshot(
|
||||
data_dir,
|
||||
¤t_manifest,
|
||||
unpacked_dir.path(),
|
||||
&snapshot_manifest,
|
||||
)?;
|
||||
```
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
```rust
|
||||
let id = 1u64;
|
||||
let vector = vec![0.1f32, 0.2, 0.3, 0.4];
|
||||
let payload = json!({"color": "red"});
|
||||
|
||||
let edge_points: Vec<PointStructPersisted> = vec![
|
||||
EdgePoint::new(
|
||||
PointId::NumId(id),
|
||||
Vectors::new_named([(VECTOR_NAME, vector.clone())]),
|
||||
payload.clone(),
|
||||
)
|
||||
.into(),
|
||||
];
|
||||
edge_shard.update(UpdateOperation::PointOperation(
|
||||
PointOperations::UpsertPoints(
|
||||
PointInsertOperations::PointsList(edge_points),
|
||||
),
|
||||
))?;
|
||||
|
||||
let server_point = PointStruct::new(
|
||||
id,
|
||||
HashMap::from([(VECTOR_NAME.to_string(), vector)]),
|
||||
payload.as_object().cloned().unwrap_or_default(),
|
||||
);
|
||||
upload_queue.push_back(server_point);
|
||||
```
|
||||
+5
-5
@@ -35,7 +35,7 @@ with tempfile.TemporaryDirectory(dir=data_dir.parent) as restore_dir:
|
||||
|
||||
EdgeShard.unpack_snapshot(str(snapshot_path), str(data_dir))
|
||||
|
||||
edge_shard = EdgeShard(SHARD_DIRECTORY)
|
||||
edge_shard = EdgeShard.load(SHARD_DIRECTORY)
|
||||
# @block-end restore-snapshot
|
||||
|
||||
# @block-start update-from-snapshot
|
||||
@@ -60,7 +60,7 @@ from pathlib import Path
|
||||
from qdrant_edge import (
|
||||
Distance,
|
||||
EdgeConfig,
|
||||
VectorDataConfig,
|
||||
EdgeVectorParams,
|
||||
)
|
||||
|
||||
SHARD_DIRECTORY = "./qdrant-edge-directory"
|
||||
@@ -69,15 +69,15 @@ VECTOR_DIMENSION=4
|
||||
|
||||
Path(SHARD_DIRECTORY).mkdir(parents=True, exist_ok=True)
|
||||
config = EdgeConfig(
|
||||
vector_data={
|
||||
VECTOR_NAME: VectorDataConfig(
|
||||
vectors={
|
||||
VECTOR_NAME: EdgeVectorParams(
|
||||
size=VECTOR_DIMENSION,
|
||||
distance=Distance.Cosine,
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
edge_shard = EdgeShard(SHARD_DIRECTORY, config)
|
||||
edge_shard = EdgeShard.create(SHARD_DIRECTORY, config)
|
||||
# @block-end initialize-edge-shard
|
||||
|
||||
# @block-start initialize-server-client
|
||||
|
||||
+199
@@ -0,0 +1,199 @@
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
|
||||
use qdrant_client::qdrant::{
|
||||
CreateCollectionBuilder, Distance, PointStruct, UpsertPointsBuilder,
|
||||
VectorParamsBuilder,
|
||||
};
|
||||
use qdrant_client::Qdrant;
|
||||
use qdrant_edge::EdgeShard;
|
||||
use qdrant_edge::internal::SnapshotManifest;
|
||||
use qdrant_edge::{
|
||||
EdgeConfig, EdgeVectorParams, PointId, PointInsertOperations,
|
||||
PointOperations, PointStruct as EdgePoint, PointStructPersisted,
|
||||
UpdateOperation, Vectors,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
pub async fn main() -> anyhow::Result<()> {
|
||||
// @hide-start
|
||||
const QDRANT_URL: &str = "";
|
||||
const QDRANT_API_KEY: &str = "";
|
||||
// @hide-end
|
||||
|
||||
// @block-start snapshot-url
|
||||
const COLLECTION_NAME: &str = "edge-collection";
|
||||
|
||||
let snapshot_url = format!(
|
||||
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot"
|
||||
);
|
||||
// @block-end snapshot-url
|
||||
|
||||
// @block-start restore-snapshot
|
||||
const SHARD_DIRECTORY: &str = "./qdrant-edge-directory";
|
||||
let data_dir = Path::new(SHARD_DIRECTORY);
|
||||
|
||||
let restore_dir = tempfile::Builder::new()
|
||||
.tempdir_in(data_dir.parent().unwrap_or(Path::new(".")))?;
|
||||
let snapshot_path = restore_dir.path().join("shard.snapshot");
|
||||
|
||||
let mut bytes = Vec::new();
|
||||
std::io::copy(
|
||||
&mut ureq::get(&snapshot_url)
|
||||
.header("api-key", QDRANT_API_KEY)
|
||||
.call()?
|
||||
.into_body()
|
||||
.into_reader(),
|
||||
&mut bytes,
|
||||
)?;
|
||||
fs_err::write(&snapshot_path, &bytes)?;
|
||||
|
||||
if data_dir.exists() {
|
||||
fs_err::remove_dir_all(data_dir)?;
|
||||
}
|
||||
fs_err::create_dir_all(data_dir)?;
|
||||
|
||||
EdgeShard::unpack_snapshot(&snapshot_path, data_dir)?;
|
||||
|
||||
let edge_shard = EdgeShard::load(data_dir, None)?;
|
||||
// @block-end restore-snapshot
|
||||
|
||||
// @block-start update-from-snapshot
|
||||
let current_manifest = edge_shard.snapshot_manifest()?;
|
||||
|
||||
let update_url = format!(
|
||||
"{QDRANT_URL}/collections/{COLLECTION_NAME}/shards/0/snapshot\
|
||||
/partial/create"
|
||||
);
|
||||
|
||||
let temp_dir = tempfile::tempdir_in(data_dir)?;
|
||||
let partial_snapshot_path = temp_dir.path().join("partial.snapshot");
|
||||
|
||||
let mut bytes = Vec::new();
|
||||
std::io::copy(
|
||||
&mut ureq::post(&update_url)
|
||||
.header("api-key", QDRANT_API_KEY)
|
||||
.send_json(¤t_manifest)?
|
||||
.into_body()
|
||||
.into_reader(),
|
||||
&mut bytes,
|
||||
)?;
|
||||
fs_err::write(&partial_snapshot_path, &bytes)?;
|
||||
|
||||
let unpacked_dir = tempfile::tempdir_in(data_dir)?;
|
||||
EdgeShard::unpack_snapshot(&partial_snapshot_path, unpacked_dir.path())?;
|
||||
let snapshot_manifest = SnapshotManifest::load_from_snapshot(
|
||||
unpacked_dir.path(),
|
||||
None,
|
||||
)?;
|
||||
|
||||
let edge_shard = EdgeShard::recover_partial_snapshot(
|
||||
data_dir,
|
||||
¤t_manifest,
|
||||
unpacked_dir.path(),
|
||||
&snapshot_manifest,
|
||||
)?;
|
||||
// @block-end update-from-snapshot
|
||||
|
||||
// @block-start initialize-edge-shard
|
||||
const VECTOR_DIMENSION: usize = 4;
|
||||
const VECTOR_NAME: &str = "my-vector";
|
||||
|
||||
fs_err::create_dir_all(SHARD_DIRECTORY)?;
|
||||
let config = EdgeConfig {
|
||||
on_disk_payload: true,
|
||||
vectors: HashMap::from([(
|
||||
VECTOR_NAME.to_string(),
|
||||
EdgeVectorParams {
|
||||
size: VECTOR_DIMENSION,
|
||||
distance: qdrant_edge::Distance::Cosine,
|
||||
on_disk: Some(true),
|
||||
quantization_config: None,
|
||||
multivector_config: None,
|
||||
datatype: None,
|
||||
hnsw_config: None,
|
||||
},
|
||||
)]),
|
||||
sparse_vectors: HashMap::new(),
|
||||
hnsw_config: Default::default(),
|
||||
quantization_config: None,
|
||||
optimizers: Default::default(),
|
||||
};
|
||||
|
||||
let edge_shard = EdgeShard::new(
|
||||
Path::new(SHARD_DIRECTORY),
|
||||
config,
|
||||
)?;
|
||||
// @block-end initialize-edge-shard
|
||||
|
||||
// @block-start initialize-server-client
|
||||
let server_client = Qdrant::from_url(QDRANT_URL)
|
||||
.api_key(QDRANT_API_KEY)
|
||||
.build()?;
|
||||
|
||||
if !server_client.collection_exists(COLLECTION_NAME).await? {
|
||||
server_client
|
||||
.create_collection(
|
||||
CreateCollectionBuilder::new(COLLECTION_NAME).vectors_config(
|
||||
VectorParamsBuilder::new(
|
||||
VECTOR_DIMENSION as u64,
|
||||
Distance::Cosine,
|
||||
),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
// @block-end initialize-server-client
|
||||
|
||||
// @block-start create-upload-queue
|
||||
// This is an in-memory queue.
|
||||
// For production use cases consider persisting changes.
|
||||
let mut upload_queue: std::collections::VecDeque<PointStruct> =
|
||||
std::collections::VecDeque::new();
|
||||
// @block-end create-upload-queue
|
||||
|
||||
// @block-start upsert-point
|
||||
let id = 1u64;
|
||||
let vector = vec![0.1f32, 0.2, 0.3, 0.4];
|
||||
let payload = json!({"color": "red"});
|
||||
|
||||
let edge_points: Vec<PointStructPersisted> = vec![
|
||||
EdgePoint::new(
|
||||
PointId::NumId(id),
|
||||
Vectors::new_named([(VECTOR_NAME, vector.clone())]),
|
||||
payload.clone(),
|
||||
)
|
||||
.into(),
|
||||
];
|
||||
edge_shard.update(UpdateOperation::PointOperation(
|
||||
PointOperations::UpsertPoints(
|
||||
PointInsertOperations::PointsList(edge_points),
|
||||
),
|
||||
))?;
|
||||
|
||||
let server_point = PointStruct::new(
|
||||
id,
|
||||
HashMap::from([(VECTOR_NAME.to_string(), vector)]),
|
||||
payload.as_object().cloned().unwrap_or_default(),
|
||||
);
|
||||
upload_queue.push_back(server_point);
|
||||
// @block-end upsert-point
|
||||
|
||||
// @block-start process-upload-queue
|
||||
const BATCH_SIZE: usize = 10;
|
||||
let points_to_upload: Vec<PointStruct> = upload_queue
|
||||
.drain(..BATCH_SIZE.min(upload_queue.len()))
|
||||
.collect();
|
||||
|
||||
if !points_to_upload.is_empty() {
|
||||
server_client
|
||||
.upsert_points(UpsertPointsBuilder::new(
|
||||
COLLECTION_NAME,
|
||||
points_to_upload,
|
||||
))
|
||||
.await?;
|
||||
}
|
||||
// @block-end process-upload-queue
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user