Add named vectors scenario to embedding migration tutorial (#2348)

* Initial commit

* Edits

* Update FAQ

* Edits

* Mention required versions
This commit is contained in:
Abdon Pijpelink
2026-05-26 09:38:24 +02:00
committed by GitHub
parent b889e19117
commit 04e3e0bb1d
52 changed files with 1967 additions and 35 deletions
@@ -18,6 +18,10 @@ public class Snippet
string OLD_MODEL = "sentence-transformers/all-minilm-l6-v2";
string NEW_MODEL = "qdrant/clip-vit-b-32-text";
string COLLECTION = "my_collection";
string OLD_VECTOR = "old-model";
string NEW_VECTOR = "new-model";
// @hide-end
// @block-start create-new-collection
@@ -150,5 +154,101 @@ public class Snippet
limit: 10
);
// @block-end search-new-collection
// @block-start add-named-vector
await client.CreateVectorNameAsync(new()
{
CollectionName = COLLECTION,
VectorName = NEW_VECTOR,
DenseConfig = new() { Size = 512, Distance = Distance.Cosine }
});
// @block-end add-named-vector
// @block-start upsert-both-vectors
await client.UpsertAsync(
collectionName: COLLECTION,
points: new List<PointStruct>
{
new()
{
Id = 1,
Vectors = new Dictionary<string, Vector>
{
[OLD_VECTOR] = new Document { Text = "Example document", Model = OLD_MODEL },
[NEW_VECTOR] = new Document { Text = "Example document", Model = NEW_MODEL },
},
Payload = { ["text"] = "Example document" }
}
}
);
// @block-end upsert-both-vectors
// @block-start re-embed-existing
PointId? reEmbedLastOffset = null;
uint reEmbedBatchSize = 100;
bool reEmbedReachedEnd = false;
while (!reEmbedReachedEnd)
{
var reEmbedScrollResult = await client.ScrollAsync(
collectionName: COLLECTION,
limit: reEmbedBatchSize,
offset: reEmbedLastOffset,
payloadSelector: true,
vectorsSelector: false
);
var reEmbedRecords = reEmbedScrollResult.Result;
reEmbedLastOffset = reEmbedScrollResult.NextPageOffset;
var pointVectors = new List<PointVectors>();
foreach (var record in reEmbedRecords)
{
var text = record.Payload.ContainsKey("text")
? record.Payload["text"].StringValue
: "";
// Update only the new vector on each point; the old vector and payload are untouched
pointVectors.Add(new PointVectors
{
Id = record.Id,
Vectors = new Dictionary<string, Vector>
{
[NEW_VECTOR] = new Document { Text = text, Model = NEW_MODEL }
}
});
}
await client.UpdateVectorsAsync(collectionName: COLLECTION, points: pointVectors);
reEmbedReachedEnd = (reEmbedLastOffset == null);
}
// @block-end re-embed-existing
// @block-start search-with-old-vector
var oldVectorResults = await client.QueryAsync(
collectionName: COLLECTION,
query: new Document { Text = "my query", Model = OLD_MODEL },
usingVector: OLD_VECTOR,
limit: 10
);
// @block-end search-with-old-vector
// @block-start search-with-new-vector
var newVectorResults = await client.QueryAsync(
collectionName: COLLECTION,
query: new Document { Text = "my query", Model = NEW_MODEL },
usingVector: NEW_VECTOR,
limit: 10
);
// @block-end search-with-new-vector
// @block-start delete-old-named-vector
await client.DeleteVectorNameAsync(new()
{
CollectionName = COLLECTION,
VectorName = OLD_VECTOR
});
// @block-end delete-old-named-vector
}
}
@@ -0,0 +1,8 @@
```csharp
await client.CreateVectorNameAsync(new()
{
CollectionName = COLLECTION,
VectorName = NEW_VECTOR,
DenseConfig = new() { Size = 512, Distance = Distance.Cosine }
});
```
@@ -0,0 +1,12 @@
```go
client.CreateVectorName(context.Background(), &qdrant.CreateVectorNameRequest{
CollectionName: COLLECTION,
VectorName: NEW_VECTOR,
VectorConfig: &qdrant.CreateVectorNameRequest_DenseConfig{
DenseConfig: &qdrant.DenseVectorCreationConfig{
Size: 512, // Size of the new embedding vectors
Distance: qdrant.Distance_Cosine,
},
},
})
```
@@ -0,0 +1,14 @@
```java
client
.createVectorNameAsync(
CreateVectorNameRequest.newBuilder()
.setCollectionName(COLLECTION)
.setVectorName(NEW_VECTOR)
.setDenseConfig(
DenseVectorCreationConfig.newBuilder()
.setSize(512) // Size of the new embedding vectors
.setDistance(Distance.Cosine) // Similarity function for the new model
.build())
.build())
.get();
```
@@ -0,0 +1,12 @@
```python
client.create_vector_name(
collection_name=COLLECTION,
vector_name=NEW_VECTOR,
vector_name_config=models.DenseVectorNameConfig(
dense=models.DenseVectorConfig(
size=512, # Size of the new embedding vectors
distance=models.Distance.COSINE # Similarity function for the new model
)
),
)
```
@@ -0,0 +1,11 @@
```rust
client
.create_vector_name(
CreateVectorNameRequestBuilder::new(
collection,
new_vector,
DenseVectorCreationConfigBuilder::new(512, Distance::Cosine), // Size of the new embedding vectors
),
)
.await?;
```
@@ -0,0 +1,8 @@
```typescript
await client.createVectorName(COLLECTION, NEW_VECTOR, {
dense: {
size: 512, // Size of the new embedding vectors
distance: "Cosine", // Similarity function for the new model
},
});
```
@@ -120,4 +120,88 @@ results = await client.QueryAsync(
},
limit: 10
);
await client.CreateVectorNameAsync(new()
{
CollectionName = COLLECTION,
VectorName = NEW_VECTOR,
DenseConfig = new() { Size = 512, Distance = Distance.Cosine }
});
await client.UpsertAsync(
collectionName: COLLECTION,
points: new List<PointStruct>
{
new()
{
Id = 1,
Vectors = new Dictionary<string, Vector>
{
[OLD_VECTOR] = new Document { Text = "Example document", Model = OLD_MODEL },
[NEW_VECTOR] = new Document { Text = "Example document", Model = NEW_MODEL },
},
Payload = { ["text"] = "Example document" }
}
}
);
PointId? reEmbedLastOffset = null;
uint reEmbedBatchSize = 100;
bool reEmbedReachedEnd = false;
while (!reEmbedReachedEnd)
{
var reEmbedScrollResult = await client.ScrollAsync(
collectionName: COLLECTION,
limit: reEmbedBatchSize,
offset: reEmbedLastOffset,
payloadSelector: true,
vectorsSelector: false
);
var reEmbedRecords = reEmbedScrollResult.Result;
reEmbedLastOffset = reEmbedScrollResult.NextPageOffset;
var pointVectors = new List<PointVectors>();
foreach (var record in reEmbedRecords)
{
var text = record.Payload.ContainsKey("text")
? record.Payload["text"].StringValue
: "";
// Update only the new vector on each point; the old vector and payload are untouched
pointVectors.Add(new PointVectors
{
Id = record.Id,
Vectors = new Dictionary<string, Vector>
{
[NEW_VECTOR] = new Document { Text = text, Model = NEW_MODEL }
}
});
}
await client.UpdateVectorsAsync(collectionName: COLLECTION, points: pointVectors);
reEmbedReachedEnd = (reEmbedLastOffset == null);
}
var oldVectorResults = await client.QueryAsync(
collectionName: COLLECTION,
query: new Document { Text = "my query", Model = OLD_MODEL },
usingVector: OLD_VECTOR,
limit: 10
);
var newVectorResults = await client.QueryAsync(
collectionName: COLLECTION,
query: new Document { Text = "my query", Model = NEW_MODEL },
usingVector: NEW_VECTOR,
limit: 10
);
await client.DeleteVectorNameAsync(new()
{
CollectionName = COLLECTION,
VectorName = OLD_VECTOR
});
```
@@ -0,0 +1,7 @@
```csharp
await client.DeleteVectorNameAsync(new()
{
CollectionName = COLLECTION,
VectorName = OLD_VECTOR
});
```
@@ -0,0 +1,6 @@
```go
client.DeleteVectorName(context.Background(), &qdrant.DeleteVectorNameRequest{
CollectionName: COLLECTION,
VectorName: OLD_VECTOR,
})
```
@@ -0,0 +1,9 @@
```java
client
.deleteVectorNameAsync(
DeleteVectorNameRequest.newBuilder()
.setCollectionName(COLLECTION)
.setVectorName(OLD_VECTOR)
.build())
.get();
```
@@ -0,0 +1,6 @@
```python
client.delete_vector_name(
collection_name=COLLECTION,
vector_name=OLD_VECTOR,
)
```
@@ -0,0 +1,8 @@
```rust
client
.delete_vector_name(DeleteVectorNameRequestBuilder::new(
collection,
old_vector,
))
.await?;
```
@@ -0,0 +1,3 @@
```typescript
await client.deleteVectorName(COLLECTION, OLD_VECTOR);
```
@@ -111,4 +111,102 @@ results, err = client.Query(context.Background(), &qdrant.QueryPoints{
}),
Limit: qdrant.PtrOf(uint64(10)),
})
client.CreateVectorName(context.Background(), &qdrant.CreateVectorNameRequest{
CollectionName: COLLECTION,
VectorName: NEW_VECTOR,
VectorConfig: &qdrant.CreateVectorNameRequest_DenseConfig{
DenseConfig: &qdrant.DenseVectorCreationConfig{
Size: 512, // Size of the new embedding vectors
Distance: qdrant.Distance_Cosine,
},
},
})
client.Upsert(context.Background(), &qdrant.UpsertPoints{
CollectionName: COLLECTION,
Points: []*qdrant.PointStruct{
{
Id: qdrant.NewIDNum(1),
Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{
OLD_VECTOR: qdrant.NewVectorDocument(&qdrant.Document{
Text: "Example document",
Model: OLD_MODEL,
}),
NEW_VECTOR: qdrant.NewVectorDocument(&qdrant.Document{
Text: "Example document",
Model: NEW_MODEL,
}),
}),
Payload: qdrant.NewValueMap(map[string]any{"text": "Example document"}),
},
},
})
var reEmbedLastOffset *qdrant.PointId
reEmbedBatchSize := uint32(100)
reEmbedReachedEnd := false
for !reEmbedReachedEnd {
reEmbedScrollResult, err := client.Scroll(context.Background(), &qdrant.ScrollPoints{
CollectionName: COLLECTION,
Limit: qdrant.PtrOf(reEmbedBatchSize),
Offset: reEmbedLastOffset,
WithPayload: qdrant.NewWithPayload(true),
WithVectors: qdrant.NewWithVectors(false),
})
reEmbedRecords := reEmbedScrollResult
pointVectors := make([]*qdrant.PointVectors, len(reEmbedRecords))
for idx, record := range reEmbedRecords {
text := ""
if val, ok := record.Payload["text"]; ok {
text = val.GetStringValue()
}
// Update only the new vector on each point; the old vector and payload are untouched
pointVectors[idx] = &qdrant.PointVectors{
Id: record.Id,
Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{
NEW_VECTOR: qdrant.NewVectorDocument(&qdrant.Document{
Text: text,
Model: NEW_MODEL,
}),
}),
}
}
client.UpdateVectors(context.Background(), &qdrant.UpdatePointVectors{
CollectionName: COLLECTION,
Points: pointVectors,
})
reEmbedReachedEnd = (reEmbedLastOffset == nil)
}
oldVectorResults, err := client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: COLLECTION,
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: "my query",
Model: OLD_MODEL,
}),
Using: qdrant.PtrOf(OLD_VECTOR),
Limit: qdrant.PtrOf(uint64(10)),
})
newVectorResults, err := client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: COLLECTION,
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: "my query",
Model: NEW_MODEL,
}),
Using: qdrant.PtrOf(NEW_VECTOR),
Limit: qdrant.PtrOf(uint64(10)),
})
client.DeleteVectorName(context.Background(), &qdrant.DeleteVectorNameRequest{
CollectionName: COLLECTION,
VectorName: OLD_VECTOR,
})
```
@@ -3,6 +3,7 @@ import static io.qdrant.client.PointIdFactory.id;
import static io.qdrant.client.QueryFactory.nearest;
import static io.qdrant.client.ValueFactory.value;
import static io.qdrant.client.VectorFactory.vector;
import static io.qdrant.client.VectorsFactory.namedVectors;
import static io.qdrant.client.VectorsFactory.vectors;
import io.qdrant.client.WithPayloadSelectorFactory;
import io.qdrant.client.WithVectorsSelectorFactory;
@@ -15,6 +16,10 @@ import io.qdrant.client.grpc.JsonWithInt.Value;
import io.qdrant.client.grpc.Points.Document;
import io.qdrant.client.grpc.Points.PointStruct;
import io.qdrant.client.grpc.Points.QueryPoints;
import io.qdrant.client.grpc.Points.CreateVectorNameRequest;
import io.qdrant.client.grpc.Points.DeleteVectorNameRequest;
import io.qdrant.client.grpc.Points.DenseVectorCreationConfig;
import io.qdrant.client.grpc.Points.PointVectors;
import io.qdrant.client.grpc.Points.UpsertPoints;
import io.qdrant.client.grpc.Points.ScrollPoints;
import io.qdrant.client.grpc.Points.UpdateMode;
@@ -141,4 +146,113 @@ QueryPoints newRequest =
.build();
results = client.queryAsync(newRequest).get();
client
.createVectorNameAsync(
CreateVectorNameRequest.newBuilder()
.setCollectionName(COLLECTION)
.setVectorName(NEW_VECTOR)
.setDenseConfig(
DenseVectorCreationConfig.newBuilder()
.setSize(512) // Size of the new embedding vectors
.setDistance(Distance.Cosine) // Similarity function for the new model
.build())
.build())
.get();
client.upsertAsync(COLLECTION, List.of(
PointStruct.newBuilder()
.setId(id(1))
.setVectors(
namedVectors(
Map.of(
OLD_VECTOR, vector(
Document.newBuilder()
.setText("Example document")
.setModel(OLD_MODEL)
.build()),
NEW_VECTOR, vector(
Document.newBuilder()
.setText("Example document")
.setModel(NEW_MODEL)
.build()))))
.putAllPayload(Map.of("text", value("Example document")))
.build())).get();
int reEmbedBatchSize = 100;
boolean reEmbedReachedEnd = false;
var reEmbedScrollBuilder = ScrollPoints.newBuilder()
.setCollectionName(COLLECTION)
.setLimit(reEmbedBatchSize)
.setWithPayload(WithPayloadSelectorFactory.enable(true))
.setWithVectors(WithVectorsSelectorFactory.enable(false));
while (!reEmbedReachedEnd) {
var reEmbedScrollResult = client.scrollAsync(reEmbedScrollBuilder.build()).get();
var reEmbedRecords = reEmbedScrollResult.getResultList();
List<PointVectors> pointVectors = new ArrayList<>();
for (var record : reEmbedRecords) {
String text = record.getPayloadMap().containsKey("text")
? record.getPayloadMap().get("text").getStringValue()
: "";
// Update only the new vector on each point; the old vector and payload are untouched
pointVectors.add(
PointVectors.newBuilder()
.setId(record.getId())
.setVectors(
namedVectors(
Map.of(
NEW_VECTOR, vector(
Document.newBuilder()
.setText(text)
.setModel(NEW_MODEL)
.build()))))
.build());
}
client.updateVectorsAsync(COLLECTION, pointVectors).get();
if (reEmbedScrollResult.hasNextPageOffset()) {
reEmbedScrollBuilder.setOffset(reEmbedScrollResult.getNextPageOffset());
} else {
reEmbedReachedEnd = true;
}
}
var oldVectorResults = client.queryAsync(
QueryPoints.newBuilder()
.setCollectionName(COLLECTION)
.setQuery(
nearest(
Document.newBuilder()
.setText("my query")
.setModel(OLD_MODEL)
.build()))
.setUsing(OLD_VECTOR)
.setLimit(10)
.build()).get();
var newVectorResults = client.queryAsync(
QueryPoints.newBuilder()
.setCollectionName(COLLECTION)
.setQuery(
nearest(
Document.newBuilder()
.setText("my query")
.setModel(NEW_MODEL)
.build()))
.setUsing(NEW_VECTOR)
.setLimit(10)
.build()).get();
client
.deleteVectorNameAsync(
DeleteVectorNameRequest.newBuilder()
.setCollectionName(COLLECTION)
.setVectorName(OLD_VECTOR)
.build())
.get();
```
@@ -95,4 +95,86 @@ results = client.query_points(
query=models.Document(text="my query", model=NEW_MODEL),
limit=10,
)
client.create_vector_name(
collection_name=COLLECTION,
vector_name=NEW_VECTOR,
vector_name_config=models.DenseVectorNameConfig(
dense=models.DenseVectorConfig(
size=512, # Size of the new embedding vectors
distance=models.Distance.COSINE # Similarity function for the new model
)
),
)
client.upsert(
collection_name=COLLECTION,
points=[
models.PointStruct(
id=1,
vector={
OLD_VECTOR: models.Document(
text="Example document",
model=OLD_MODEL,
),
NEW_VECTOR: models.Document(
text="Example document",
model=NEW_MODEL,
),
},
payload={"text": "Example document"}
)
]
)
last_offset = None
batch_size = 100
reached_end = False
while not reached_end:
records, last_offset = client.scroll(
collection_name=COLLECTION,
limit=batch_size,
offset=last_offset,
with_payload=True,
with_vectors=False,
)
# Update only the new vector on each point; the old vector and payload are untouched
client.update_vectors(
collection_name=COLLECTION,
points=[
models.PointVectors(
id=record.id,
vector={
NEW_VECTOR: models.Document(
text=(record.payload or {}).get("text", ""),
model=NEW_MODEL,
)
},
)
for record in records
],
)
reached_end = last_offset is None
results = client.query_points(
collection_name=COLLECTION,
query=models.Document(text="my query", model=OLD_MODEL),
using=OLD_VECTOR,
limit=10,
)
results = client.query_points(
collection_name=COLLECTION,
query=models.Document(text="my query", model=NEW_MODEL),
using=NEW_VECTOR,
limit=10,
)
client.delete_vector_name(
collection_name=COLLECTION,
vector_name=OLD_VECTOR,
)
```
@@ -0,0 +1,41 @@
```csharp
PointId? reEmbedLastOffset = null;
uint reEmbedBatchSize = 100;
bool reEmbedReachedEnd = false;
while (!reEmbedReachedEnd)
{
var reEmbedScrollResult = await client.ScrollAsync(
collectionName: COLLECTION,
limit: reEmbedBatchSize,
offset: reEmbedLastOffset,
payloadSelector: true,
vectorsSelector: false
);
var reEmbedRecords = reEmbedScrollResult.Result;
reEmbedLastOffset = reEmbedScrollResult.NextPageOffset;
var pointVectors = new List<PointVectors>();
foreach (var record in reEmbedRecords)
{
var text = record.Payload.ContainsKey("text")
? record.Payload["text"].StringValue
: "";
// Update only the new vector on each point; the old vector and payload are untouched
pointVectors.Add(new PointVectors
{
Id = record.Id,
Vectors = new Dictionary<string, Vector>
{
[NEW_VECTOR] = new Document { Text = text, Model = NEW_MODEL }
}
});
}
await client.UpdateVectorsAsync(collectionName: COLLECTION, points: pointVectors);
reEmbedReachedEnd = (reEmbedLastOffset == null);
}
```
@@ -0,0 +1,43 @@
```go
var reEmbedLastOffset *qdrant.PointId
reEmbedBatchSize := uint32(100)
reEmbedReachedEnd := false
for !reEmbedReachedEnd {
reEmbedScrollResult, err := client.Scroll(context.Background(), &qdrant.ScrollPoints{
CollectionName: COLLECTION,
Limit: qdrant.PtrOf(reEmbedBatchSize),
Offset: reEmbedLastOffset,
WithPayload: qdrant.NewWithPayload(true),
WithVectors: qdrant.NewWithVectors(false),
})
reEmbedRecords := reEmbedScrollResult
pointVectors := make([]*qdrant.PointVectors, len(reEmbedRecords))
for idx, record := range reEmbedRecords {
text := ""
if val, ok := record.Payload["text"]; ok {
text = val.GetStringValue()
}
// Update only the new vector on each point; the old vector and payload are untouched
pointVectors[idx] = &qdrant.PointVectors{
Id: record.Id,
Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{
NEW_VECTOR: qdrant.NewVectorDocument(&qdrant.Document{
Text: text,
Model: NEW_MODEL,
}),
}),
}
}
client.UpdateVectors(context.Background(), &qdrant.UpdatePointVectors{
CollectionName: COLLECTION,
Points: pointVectors,
})
reEmbedReachedEnd = (reEmbedLastOffset == nil)
}
```
@@ -0,0 +1,44 @@
```java
int reEmbedBatchSize = 100;
boolean reEmbedReachedEnd = false;
var reEmbedScrollBuilder = ScrollPoints.newBuilder()
.setCollectionName(COLLECTION)
.setLimit(reEmbedBatchSize)
.setWithPayload(WithPayloadSelectorFactory.enable(true))
.setWithVectors(WithVectorsSelectorFactory.enable(false));
while (!reEmbedReachedEnd) {
var reEmbedScrollResult = client.scrollAsync(reEmbedScrollBuilder.build()).get();
var reEmbedRecords = reEmbedScrollResult.getResultList();
List<PointVectors> pointVectors = new ArrayList<>();
for (var record : reEmbedRecords) {
String text = record.getPayloadMap().containsKey("text")
? record.getPayloadMap().get("text").getStringValue()
: "";
// Update only the new vector on each point; the old vector and payload are untouched
pointVectors.add(
PointVectors.newBuilder()
.setId(record.getId())
.setVectors(
namedVectors(
Map.of(
NEW_VECTOR, vector(
Document.newBuilder()
.setText(text)
.setModel(NEW_MODEL)
.build()))))
.build());
}
client.updateVectorsAsync(COLLECTION, pointVectors).get();
if (reEmbedScrollResult.hasNextPageOffset()) {
reEmbedScrollBuilder.setOffset(reEmbedScrollResult.getNextPageOffset());
} else {
reEmbedReachedEnd = true;
}
}
```
@@ -0,0 +1,33 @@
```python
last_offset = None
batch_size = 100
reached_end = False
while not reached_end:
records, last_offset = client.scroll(
collection_name=COLLECTION,
limit=batch_size,
offset=last_offset,
with_payload=True,
with_vectors=False,
)
# Update only the new vector on each point; the old vector and payload are untouched
client.update_vectors(
collection_name=COLLECTION,
points=[
models.PointVectors(
id=record.id,
vector={
NEW_VECTOR: models.Document(
text=(record.payload or {}).get("text", ""),
model=NEW_MODEL,
)
},
)
for record in records
],
)
reached_end = last_offset is None
```
@@ -0,0 +1,47 @@
```rust
let mut last_offset = None;
let batch_size = 100;
loop {
let mut scroll_builder = ScrollPointsBuilder::new(collection)
.limit(batch_size)
.with_payload(true)
.with_vectors(false);
if let Some(offset) = last_offset {
scroll_builder = scroll_builder.offset(offset);
}
let scroll_result = client.scroll(scroll_builder).await?;
let records = scroll_result.result;
last_offset = scroll_result.next_page_offset;
// Update only the new vector on each point; the old vector and payload are untouched
let point_vectors: Vec<PointVectors> = records
.iter()
.map(|record| PointVectors {
id: record.id.clone(),
vectors: Some(
HashMap::<String, Document>::from([(
new_vector.to_string(),
Document::new(
record.payload.get("text")
.and_then(|v| v.as_str())
.map_or("", |v| v),
new_model,
),
)])
.into(),
),
})
.collect();
client
.update_vectors(UpdatePointVectorsBuilder::new(collection, point_vectors))
.await?;
if last_offset.is_none() {
break;
}
}
```
@@ -0,0 +1,32 @@
```typescript
let reEmbedLastOffset: number | string | undefined = undefined;
const reEmbedBatchSize = 100;
let reEmbedReachedEnd = false;
while (!reEmbedReachedEnd) {
const reEmbedScrollResult = await client.scroll(COLLECTION, {
limit: reEmbedBatchSize,
offset: reEmbedLastOffset,
with_payload: true,
with_vector: false,
});
const records = reEmbedScrollResult.points;
reEmbedLastOffset = reEmbedScrollResult.next_page_offset as number | string | undefined;
// Update only the new vector on each point; the old vector and payload are untouched
await client.updateVectors(COLLECTION, {
points: records.map((record) => ({
id: record.id,
vector: {
[NEW_VECTOR]: {
text: ((record.payload?.text as string) ?? ""),
model: NEW_MODEL,
},
},
})),
});
reEmbedReachedEnd = reEmbedLastOffset == null;
}
```
@@ -1,7 +1,11 @@
```rust
use std::collections::HashMap;
use qdrant_client::qdrant::{
CreateCollectionBuilder, Distance, Document, PointStruct, Query, QueryPointsBuilder,
ScrollPointsBuilder, UpdateMode, UpsertPointsBuilder, VectorParamsBuilder,
CreateCollectionBuilder, CreateVectorNameRequestBuilder, DeleteVectorNameRequestBuilder,
DenseVectorCreationConfigBuilder, Distance, Document, NamedVectors, PointStruct, PointVectors,
Query, QueryPointsBuilder, ScrollPointsBuilder, UpdateMode, UpdatePointVectorsBuilder,
UpsertPointsBuilder, VectorParamsBuilder,
};
use qdrant_client::Qdrant;
@@ -107,4 +111,112 @@ let results = client
.limit(10),
)
.await?;
client
.create_vector_name(
CreateVectorNameRequestBuilder::new(
collection,
new_vector,
DenseVectorCreationConfigBuilder::new(512, Distance::Cosine), // Size of the new embedding vectors
),
)
.await?;
client
.upsert_points(UpsertPointsBuilder::new(
collection,
vec![PointStruct::new(
1,
NamedVectors::default()
.add_vector(
old_vector,
Document {
text: "Example document".into(),
model: old_model.into(),
..Default::default()
},
)
.add_vector(
new_vector,
Document {
text: "Example document".into(),
model: new_model.into(),
..Default::default()
},
),
[("text", "Example document".into())],
)],
))
.await?;
let mut last_offset = None;
let batch_size = 100;
loop {
let mut scroll_builder = ScrollPointsBuilder::new(collection)
.limit(batch_size)
.with_payload(true)
.with_vectors(false);
if let Some(offset) = last_offset {
scroll_builder = scroll_builder.offset(offset);
}
let scroll_result = client.scroll(scroll_builder).await?;
let records = scroll_result.result;
last_offset = scroll_result.next_page_offset;
// Update only the new vector on each point; the old vector and payload are untouched
let point_vectors: Vec<PointVectors> = records
.iter()
.map(|record| PointVectors {
id: record.id.clone(),
vectors: Some(
HashMap::<String, Document>::from([(
new_vector.to_string(),
Document::new(
record.payload.get("text")
.and_then(|v| v.as_str())
.map_or("", |v| v),
new_model,
),
)])
.into(),
),
})
.collect();
client
.update_vectors(UpdatePointVectorsBuilder::new(collection, point_vectors))
.await?;
if last_offset.is_none() {
break;
}
}
let old_vector_results = client
.query(
QueryPointsBuilder::new(collection)
.query(Query::new_nearest(Document::new("my query", old_model)))
.using(old_vector)
.limit(10),
)
.await?;
let new_vector_results = client
.query(
QueryPointsBuilder::new(collection)
.query(Query::new_nearest(Document::new("my query", new_model)))
.using(new_vector)
.limit(10),
)
.await?;
client
.delete_vector_name(DeleteVectorNameRequestBuilder::new(
collection,
old_vector,
))
.await?;
```
@@ -0,0 +1,8 @@
```csharp
var newVectorResults = await client.QueryAsync(
collectionName: COLLECTION,
query: new Document { Text = "my query", Model = NEW_MODEL },
usingVector: NEW_VECTOR,
limit: 10
);
```
@@ -0,0 +1,11 @@
```go
newVectorResults, err := client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: COLLECTION,
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: "my query",
Model: NEW_MODEL,
}),
Using: qdrant.PtrOf(NEW_VECTOR),
Limit: qdrant.PtrOf(uint64(10)),
})
```
@@ -0,0 +1,14 @@
```java
var newVectorResults = client.queryAsync(
QueryPoints.newBuilder()
.setCollectionName(COLLECTION)
.setQuery(
nearest(
Document.newBuilder()
.setText("my query")
.setModel(NEW_MODEL)
.build()))
.setUsing(NEW_VECTOR)
.setLimit(10)
.build()).get();
```
@@ -0,0 +1,8 @@
```python
results = client.query_points(
collection_name=COLLECTION,
query=models.Document(text="my query", model=NEW_MODEL),
using=NEW_VECTOR,
limit=10,
)
```
@@ -0,0 +1,10 @@
```rust
let new_vector_results = client
.query(
QueryPointsBuilder::new(collection)
.query(Query::new_nearest(Document::new("my query", new_model)))
.using(new_vector)
.limit(10),
)
.await?;
```
@@ -0,0 +1,10 @@
```typescript
const newVectorResults = await client.query(COLLECTION, {
query: {
text: "my query",
model: NEW_MODEL,
},
using: NEW_VECTOR,
limit: 10,
});
```
@@ -0,0 +1,8 @@
```csharp
var oldVectorResults = await client.QueryAsync(
collectionName: COLLECTION,
query: new Document { Text = "my query", Model = OLD_MODEL },
usingVector: OLD_VECTOR,
limit: 10
);
```
@@ -0,0 +1,11 @@
```go
oldVectorResults, err := client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: COLLECTION,
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: "my query",
Model: OLD_MODEL,
}),
Using: qdrant.PtrOf(OLD_VECTOR),
Limit: qdrant.PtrOf(uint64(10)),
})
```
@@ -0,0 +1,14 @@
```java
var oldVectorResults = client.queryAsync(
QueryPoints.newBuilder()
.setCollectionName(COLLECTION)
.setQuery(
nearest(
Document.newBuilder()
.setText("my query")
.setModel(OLD_MODEL)
.build()))
.setUsing(OLD_VECTOR)
.setLimit(10)
.build()).get();
```
@@ -0,0 +1,8 @@
```python
results = client.query_points(
collection_name=COLLECTION,
query=models.Document(text="my query", model=OLD_MODEL),
using=OLD_VECTOR,
limit=10,
)
```
@@ -0,0 +1,10 @@
```rust
let old_vector_results = client
.query(
QueryPointsBuilder::new(collection)
.query(Query::new_nearest(Document::new("my query", old_model)))
.using(old_vector)
.limit(10),
)
.await?;
```
@@ -0,0 +1,10 @@
```typescript
const oldVectorResults = await client.query(COLLECTION, {
query: {
text: "my query",
model: OLD_MODEL,
},
using: OLD_VECTOR,
limit: 10,
});
```
@@ -93,4 +93,81 @@ const resultsNew = await client.query(NEW_COLLECTION, {
},
limit: 10,
});
await client.createVectorName(COLLECTION, NEW_VECTOR, {
dense: {
size: 512, // Size of the new embedding vectors
distance: "Cosine", // Similarity function for the new model
},
});
await client.upsert(COLLECTION, {
points: [
{
id: 1,
vector: {
[OLD_VECTOR]: {
text: "Example document",
model: OLD_MODEL,
},
[NEW_VECTOR]: {
text: "Example document",
model: NEW_MODEL,
},
},
payload: { text: "Example document" },
},
],
});
let reEmbedLastOffset: number | string | undefined = undefined;
const reEmbedBatchSize = 100;
let reEmbedReachedEnd = false;
while (!reEmbedReachedEnd) {
const reEmbedScrollResult = await client.scroll(COLLECTION, {
limit: reEmbedBatchSize,
offset: reEmbedLastOffset,
with_payload: true,
with_vector: false,
});
const records = reEmbedScrollResult.points;
reEmbedLastOffset = reEmbedScrollResult.next_page_offset as number | string | undefined;
// Update only the new vector on each point; the old vector and payload are untouched
await client.updateVectors(COLLECTION, {
points: records.map((record) => ({
id: record.id,
vector: {
[NEW_VECTOR]: {
text: ((record.payload?.text as string) ?? ""),
model: NEW_MODEL,
},
},
})),
});
reEmbedReachedEnd = reEmbedLastOffset == null;
}
const oldVectorResults = await client.query(COLLECTION, {
query: {
text: "my query",
model: OLD_MODEL,
},
using: OLD_VECTOR,
limit: 10,
});
const newVectorResults = await client.query(COLLECTION, {
query: {
text: "my query",
model: NEW_MODEL,
},
using: NEW_VECTOR,
limit: 10,
});
await client.deleteVectorName(COLLECTION, OLD_VECTOR);
```
@@ -0,0 +1,18 @@
```csharp
await client.UpsertAsync(
collectionName: COLLECTION,
points: new List<PointStruct>
{
new()
{
Id = 1,
Vectors = new Dictionary<string, Vector>
{
[OLD_VECTOR] = new Document { Text = "Example document", Model = OLD_MODEL },
[NEW_VECTOR] = new Document { Text = "Example document", Model = NEW_MODEL },
},
Payload = { ["text"] = "Example document" }
}
}
);
```
@@ -0,0 +1,21 @@
```go
client.Upsert(context.Background(), &qdrant.UpsertPoints{
CollectionName: COLLECTION,
Points: []*qdrant.PointStruct{
{
Id: qdrant.NewIDNum(1),
Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{
OLD_VECTOR: qdrant.NewVectorDocument(&qdrant.Document{
Text: "Example document",
Model: OLD_MODEL,
}),
NEW_VECTOR: qdrant.NewVectorDocument(&qdrant.Document{
Text: "Example document",
Model: NEW_MODEL,
}),
}),
Payload: qdrant.NewValueMap(map[string]any{"text": "Example document"}),
},
},
})
```
@@ -0,0 +1,20 @@
```java
client.upsertAsync(COLLECTION, List.of(
PointStruct.newBuilder()
.setId(id(1))
.setVectors(
namedVectors(
Map.of(
OLD_VECTOR, vector(
Document.newBuilder()
.setText("Example document")
.setModel(OLD_MODEL)
.build()),
NEW_VECTOR, vector(
Document.newBuilder()
.setText("Example document")
.setModel(NEW_MODEL)
.build()))))
.putAllPayload(Map.of("text", value("Example document")))
.build())).get();
```
@@ -0,0 +1,21 @@
```python
client.upsert(
collection_name=COLLECTION,
points=[
models.PointStruct(
id=1,
vector={
OLD_VECTOR: models.Document(
text="Example document",
model=OLD_MODEL,
),
NEW_VECTOR: models.Document(
text="Example document",
model=NEW_MODEL,
),
},
payload={"text": "Example document"}
)
]
)
```
@@ -0,0 +1,28 @@
```rust
client
.upsert_points(UpsertPointsBuilder::new(
collection,
vec![PointStruct::new(
1,
NamedVectors::default()
.add_vector(
old_vector,
Document {
text: "Example document".into(),
model: old_model.into(),
..Default::default()
},
)
.add_vector(
new_vector,
Document {
text: "Example document".into(),
model: new_model.into(),
..Default::default()
},
),
[("text", "Example document".into())],
)],
))
.await?;
```
@@ -0,0 +1,20 @@
```typescript
await client.upsert(COLLECTION, {
points: [
{
id: 1,
vector: {
[OLD_VECTOR]: {
text: "Example document",
model: OLD_MODEL,
},
[NEW_VECTOR]: {
text: "Example document",
model: NEW_MODEL,
},
},
payload: { text: "Example document" },
},
],
});
```
@@ -23,6 +23,10 @@ func Main() {
OLD_MODEL := "sentence-transformers/all-minilm-l6-v2"
NEW_MODEL := "qdrant/clip-vit-b-32-text"
COLLECTION := "my_collection"
OLD_VECTOR := "old-model"
NEW_VECTOR := "new-model"
// @hide-end
// @block-start create-new-collection
@@ -164,4 +168,135 @@ func Main() {
}
_ = results
// @hide-end
// @block-start add-named-vector
client.CreateVectorName(context.Background(), &qdrant.CreateVectorNameRequest{
CollectionName: COLLECTION,
VectorName: NEW_VECTOR,
VectorConfig: &qdrant.CreateVectorNameRequest_DenseConfig{
DenseConfig: &qdrant.DenseVectorCreationConfig{
Size: 512, // Size of the new embedding vectors
Distance: qdrant.Distance_Cosine,
},
},
})
// @block-end add-named-vector
// @block-start upsert-both-vectors
client.Upsert(context.Background(), &qdrant.UpsertPoints{
CollectionName: COLLECTION,
Points: []*qdrant.PointStruct{
{
Id: qdrant.NewIDNum(1),
Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{
OLD_VECTOR: qdrant.NewVectorDocument(&qdrant.Document{
Text: "Example document",
Model: OLD_MODEL,
}),
NEW_VECTOR: qdrant.NewVectorDocument(&qdrant.Document{
Text: "Example document",
Model: NEW_MODEL,
}),
}),
Payload: qdrant.NewValueMap(map[string]any{"text": "Example document"}),
},
},
})
// @block-end upsert-both-vectors
// @block-start re-embed-existing
var reEmbedLastOffset *qdrant.PointId
reEmbedBatchSize := uint32(100)
reEmbedReachedEnd := false
for !reEmbedReachedEnd {
reEmbedScrollResult, err := client.Scroll(context.Background(), &qdrant.ScrollPoints{
CollectionName: COLLECTION,
Limit: qdrant.PtrOf(reEmbedBatchSize),
Offset: reEmbedLastOffset,
WithPayload: qdrant.NewWithPayload(true),
WithVectors: qdrant.NewWithVectors(false),
})
// @hide-start
if err != nil {
panic(err)
}
// @hide-end
reEmbedRecords := reEmbedScrollResult
reEmbedLastOffset = reEmbedScrollResult[len(reEmbedScrollResult)-1].Id // @hide
pointVectors := make([]*qdrant.PointVectors, len(reEmbedRecords))
for idx, record := range reEmbedRecords {
text := ""
if val, ok := record.Payload["text"]; ok {
text = val.GetStringValue()
}
// Update only the new vector on each point; the old vector and payload are untouched
pointVectors[idx] = &qdrant.PointVectors{
Id: record.Id,
Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{
NEW_VECTOR: qdrant.NewVectorDocument(&qdrant.Document{
Text: text,
Model: NEW_MODEL,
}),
}),
}
}
client.UpdateVectors(context.Background(), &qdrant.UpdatePointVectors{
CollectionName: COLLECTION,
Points: pointVectors,
})
reEmbedReachedEnd = (reEmbedLastOffset == nil)
}
// @block-end re-embed-existing
// @block-start search-with-old-vector
oldVectorResults, err := client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: COLLECTION,
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: "my query",
Model: OLD_MODEL,
}),
Using: qdrant.PtrOf(OLD_VECTOR),
Limit: qdrant.PtrOf(uint64(10)),
})
// @block-end search-with-old-vector
// @hide-start
if err != nil {
panic(err)
}
_ = oldVectorResults
// @hide-end
// @block-start search-with-new-vector
newVectorResults, err := client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: COLLECTION,
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: "my query",
Model: NEW_MODEL,
}),
Using: qdrant.PtrOf(NEW_VECTOR),
Limit: qdrant.PtrOf(uint64(10)),
})
// @block-end search-with-new-vector
// @hide-start
if err != nil {
panic(err)
}
_ = newVectorResults
// @hide-end
// @block-start delete-old-named-vector
client.DeleteVectorName(context.Background(), &qdrant.DeleteVectorNameRequest{
CollectionName: COLLECTION,
VectorName: OLD_VECTOR,
})
// @block-end delete-old-named-vector
}
@@ -4,6 +4,7 @@ import static io.qdrant.client.PointIdFactory.id;
import static io.qdrant.client.QueryFactory.nearest;
import static io.qdrant.client.ValueFactory.value;
import static io.qdrant.client.VectorFactory.vector;
import static io.qdrant.client.VectorsFactory.namedVectors;
import static io.qdrant.client.VectorsFactory.vectors;
import io.qdrant.client.WithPayloadSelectorFactory;
import io.qdrant.client.WithVectorsSelectorFactory;
@@ -16,6 +17,10 @@ import io.qdrant.client.grpc.JsonWithInt.Value;
import io.qdrant.client.grpc.Points.Document;
import io.qdrant.client.grpc.Points.PointStruct;
import io.qdrant.client.grpc.Points.QueryPoints;
import io.qdrant.client.grpc.Points.CreateVectorNameRequest;
import io.qdrant.client.grpc.Points.DeleteVectorNameRequest;
import io.qdrant.client.grpc.Points.DenseVectorCreationConfig;
import io.qdrant.client.grpc.Points.PointVectors;
import io.qdrant.client.grpc.Points.UpsertPoints;
import io.qdrant.client.grpc.Points.ScrollPoints;
import io.qdrant.client.grpc.Points.UpdateMode;
@@ -41,6 +46,10 @@ public class Snippet {
String OLD_MODEL = "sentence-transformers/all-minilm-l6-v2";
String NEW_MODEL = "qdrant/clip-vit-b-32-text";
String COLLECTION = "my_collection";
String OLD_VECTOR = "old-model";
String NEW_VECTOR = "new-model";
// @hide-end
// @block-start create-new-collection
@@ -174,6 +183,127 @@ public class Snippet {
results = client.queryAsync(newRequest).get();
// @block-end search-new-collection
// @block-start add-named-vector
client
.createVectorNameAsync(
CreateVectorNameRequest.newBuilder()
.setCollectionName(COLLECTION)
.setVectorName(NEW_VECTOR)
.setDenseConfig(
DenseVectorCreationConfig.newBuilder()
.setSize(512) // Size of the new embedding vectors
.setDistance(Distance.Cosine) // Similarity function for the new model
.build())
.build())
.get();
// @block-end add-named-vector
// @block-start upsert-both-vectors
client.upsertAsync(COLLECTION, List.of(
PointStruct.newBuilder()
.setId(id(1))
.setVectors(
namedVectors(
Map.of(
OLD_VECTOR, vector(
Document.newBuilder()
.setText("Example document")
.setModel(OLD_MODEL)
.build()),
NEW_VECTOR, vector(
Document.newBuilder()
.setText("Example document")
.setModel(NEW_MODEL)
.build()))))
.putAllPayload(Map.of("text", value("Example document")))
.build())).get();
// @block-end upsert-both-vectors
// @block-start re-embed-existing
int reEmbedBatchSize = 100;
boolean reEmbedReachedEnd = false;
var reEmbedScrollBuilder = ScrollPoints.newBuilder()
.setCollectionName(COLLECTION)
.setLimit(reEmbedBatchSize)
.setWithPayload(WithPayloadSelectorFactory.enable(true))
.setWithVectors(WithVectorsSelectorFactory.enable(false));
while (!reEmbedReachedEnd) {
var reEmbedScrollResult = client.scrollAsync(reEmbedScrollBuilder.build()).get();
var reEmbedRecords = reEmbedScrollResult.getResultList();
List<PointVectors> pointVectors = new ArrayList<>();
for (var record : reEmbedRecords) {
String text = record.getPayloadMap().containsKey("text")
? record.getPayloadMap().get("text").getStringValue()
: "";
// Update only the new vector on each point; the old vector and payload are untouched
pointVectors.add(
PointVectors.newBuilder()
.setId(record.getId())
.setVectors(
namedVectors(
Map.of(
NEW_VECTOR, vector(
Document.newBuilder()
.setText(text)
.setModel(NEW_MODEL)
.build()))))
.build());
}
client.updateVectorsAsync(COLLECTION, pointVectors).get();
if (reEmbedScrollResult.hasNextPageOffset()) {
reEmbedScrollBuilder.setOffset(reEmbedScrollResult.getNextPageOffset());
} else {
reEmbedReachedEnd = true;
}
}
// @block-end re-embed-existing
// @block-start search-with-old-vector
var oldVectorResults = client.queryAsync(
QueryPoints.newBuilder()
.setCollectionName(COLLECTION)
.setQuery(
nearest(
Document.newBuilder()
.setText("my query")
.setModel(OLD_MODEL)
.build()))
.setUsing(OLD_VECTOR)
.setLimit(10)
.build()).get();
// @block-end search-with-old-vector
// @block-start search-with-new-vector
var newVectorResults = client.queryAsync(
QueryPoints.newBuilder()
.setCollectionName(COLLECTION)
.setQuery(
nearest(
Document.newBuilder()
.setText("my query")
.setModel(NEW_MODEL)
.build()))
.setUsing(NEW_VECTOR)
.setLimit(10)
.build()).get();
// @block-end search-with-new-vector
// @block-start delete-old-named-vector
client
.deleteVectorNameAsync(
DeleteVectorNameRequest.newBuilder()
.setCollectionName(COLLECTION)
.setVectorName(OLD_VECTOR)
.build())
.get();
// @block-end delete-old-named-vector
}
}
@@ -11,6 +11,10 @@ OLD_COLLECTION="old_collection"
OLD_MODEL="sentence-transformers/all-minilm-l6-v2"
NEW_MODEL="qdrant/clip-vit-b-32-text"
COLLECTION="my_collection"
OLD_VECTOR="old-model"
NEW_VECTOR="new-model"
# @hide-end
# @block-start create-new-collection
@@ -119,3 +123,97 @@ results = client.query_points(
limit=10,
)
# @block-end search-new-collection
# @block-start add-named-vector
client.create_vector_name(
collection_name=COLLECTION,
vector_name=NEW_VECTOR,
vector_name_config=models.DenseVectorNameConfig(
dense=models.DenseVectorConfig(
size=512, # Size of the new embedding vectors
distance=models.Distance.COSINE # Similarity function for the new model
)
),
)
# @block-end add-named-vector
# @block-start upsert-both-vectors
client.upsert(
collection_name=COLLECTION,
points=[
models.PointStruct(
id=1,
vector={
OLD_VECTOR: models.Document(
text="Example document",
model=OLD_MODEL,
),
NEW_VECTOR: models.Document(
text="Example document",
model=NEW_MODEL,
),
},
payload={"text": "Example document"}
)
]
)
# @block-end upsert-both-vectors
# @block-start re-embed-existing
last_offset = None
batch_size = 100
reached_end = False
while not reached_end:
records, last_offset = client.scroll(
collection_name=COLLECTION,
limit=batch_size,
offset=last_offset,
with_payload=True,
with_vectors=False,
)
# Update only the new vector on each point; the old vector and payload are untouched
client.update_vectors(
collection_name=COLLECTION,
points=[
models.PointVectors(
id=record.id,
vector={
NEW_VECTOR: models.Document(
text=(record.payload or {}).get("text", ""),
model=NEW_MODEL,
)
},
)
for record in records
],
)
reached_end = last_offset is None
# @block-end re-embed-existing
# @block-start search-with-old-vector
results = client.query_points(
collection_name=COLLECTION,
query=models.Document(text="my query", model=OLD_MODEL),
using=OLD_VECTOR,
limit=10,
)
# @block-end search-with-old-vector
# @block-start search-with-new-vector
results = client.query_points(
collection_name=COLLECTION,
query=models.Document(text="my query", model=NEW_MODEL),
using=NEW_VECTOR,
limit=10,
)
# @block-end search-with-new-vector
# @block-start delete-old-named-vector
client.delete_vector_name(
collection_name=COLLECTION,
vector_name=OLD_VECTOR,
)
# @block-end delete-old-named-vector
@@ -1,6 +1,10 @@
use std::collections::HashMap;
use qdrant_client::qdrant::{
CreateCollectionBuilder, Distance, Document, PointStruct, Query, QueryPointsBuilder,
ScrollPointsBuilder, UpdateMode, UpsertPointsBuilder, VectorParamsBuilder,
CreateCollectionBuilder, CreateVectorNameRequestBuilder, DeleteVectorNameRequestBuilder,
DenseVectorCreationConfigBuilder, Distance, Document, NamedVectors, PointStruct, PointVectors,
Query, QueryPointsBuilder, ScrollPointsBuilder, UpdateMode, UpdatePointVectorsBuilder,
UpsertPointsBuilder, VectorParamsBuilder,
};
use qdrant_client::Qdrant;
@@ -18,6 +22,10 @@ pub async fn main() -> anyhow::Result<()> {
let old_model = "sentence-transformers/all-minilm-l6-v2";
let new_model = "qdrant/clip-vit-b-32-text";
let collection = "my_collection";
let old_vector = "old-model";
let new_vector = "new-model";
// @hide-end
// @block-start create-new-collection
@@ -143,5 +151,133 @@ pub async fn main() -> anyhow::Result<()> {
_ = results;
// @hide-end
// @block-start add-named-vector
client
.create_vector_name(
CreateVectorNameRequestBuilder::new(
collection,
new_vector,
DenseVectorCreationConfigBuilder::new(512, Distance::Cosine), // Size of the new embedding vectors
),
)
.await?;
// @block-end add-named-vector
// @block-start upsert-both-vectors
client
.upsert_points(UpsertPointsBuilder::new(
collection,
vec![PointStruct::new(
1,
NamedVectors::default()
.add_vector(
old_vector,
Document {
text: "Example document".into(),
model: old_model.into(),
..Default::default()
},
)
.add_vector(
new_vector,
Document {
text: "Example document".into(),
model: new_model.into(),
..Default::default()
},
),
[("text", "Example document".into())],
)],
))
.await?;
// @block-end upsert-both-vectors
// @block-start re-embed-existing
let mut last_offset = None;
let batch_size = 100;
loop {
let mut scroll_builder = ScrollPointsBuilder::new(collection)
.limit(batch_size)
.with_payload(true)
.with_vectors(false);
if let Some(offset) = last_offset {
scroll_builder = scroll_builder.offset(offset);
}
let scroll_result = client.scroll(scroll_builder).await?;
let records = scroll_result.result;
last_offset = scroll_result.next_page_offset;
// Update only the new vector on each point; the old vector and payload are untouched
let point_vectors: Vec<PointVectors> = records
.iter()
.map(|record| PointVectors {
id: record.id.clone(),
vectors: Some(
HashMap::<String, Document>::from([(
new_vector.to_string(),
Document::new(
record.payload.get("text")
.and_then(|v| v.as_str())
.map_or("", |v| v),
new_model,
),
)])
.into(),
),
})
.collect();
client
.update_vectors(UpdatePointVectorsBuilder::new(collection, point_vectors))
.await?;
if last_offset.is_none() {
break;
}
}
// @block-end re-embed-existing
// @block-start search-with-old-vector
let old_vector_results = client
.query(
QueryPointsBuilder::new(collection)
.query(Query::new_nearest(Document::new("my query", old_model)))
.using(old_vector)
.limit(10),
)
.await?;
// @block-end search-with-old-vector
// @hide-start
_ = old_vector_results;
// @hide-end
// @block-start search-with-new-vector
let new_vector_results = client
.query(
QueryPointsBuilder::new(collection)
.query(Query::new_nearest(Document::new("my query", new_model)))
.using(new_vector)
.limit(10),
)
.await?;
// @block-end search-with-new-vector
// @hide-start
_ = new_vector_results;
// @hide-end
// @block-start delete-old-named-vector
client
.delete_vector_name(DeleteVectorNameRequestBuilder::new(
collection,
old_vector,
))
.await?;
// @block-end delete-old-named-vector
Ok(())
}
@@ -14,6 +14,10 @@ const OLD_COLLECTION = "old_collection";
const OLD_MODEL = "sentence-transformers/all-minilm-l6-v2";
const NEW_MODEL = "qdrant/clip-vit-b-32-text";
const COLLECTION = "my_collection";
const OLD_VECTOR = "old-model";
const NEW_VECTOR = "new-model";
// @hide-end
// @block-start create-new-collection
@@ -120,3 +124,92 @@ const resultsNew = await client.query(NEW_COLLECTION, {
limit: 10,
});
// @block-end search-new-collection
// @block-start add-named-vector
await client.createVectorName(COLLECTION, NEW_VECTOR, {
dense: {
size: 512, // Size of the new embedding vectors
distance: "Cosine", // Similarity function for the new model
},
});
// @block-end add-named-vector
// @block-start upsert-both-vectors
await client.upsert(COLLECTION, {
points: [
{
id: 1,
vector: {
[OLD_VECTOR]: {
text: "Example document",
model: OLD_MODEL,
},
[NEW_VECTOR]: {
text: "Example document",
model: NEW_MODEL,
},
},
payload: { text: "Example document" },
},
],
});
// @block-end upsert-both-vectors
// @block-start re-embed-existing
let reEmbedLastOffset: number | string | undefined = undefined;
const reEmbedBatchSize = 100;
let reEmbedReachedEnd = false;
while (!reEmbedReachedEnd) {
const reEmbedScrollResult = await client.scroll(COLLECTION, {
limit: reEmbedBatchSize,
offset: reEmbedLastOffset,
with_payload: true,
with_vector: false,
});
const records = reEmbedScrollResult.points;
reEmbedLastOffset = reEmbedScrollResult.next_page_offset as number | string | undefined;
// Update only the new vector on each point; the old vector and payload are untouched
await client.updateVectors(COLLECTION, {
points: records.map((record) => ({
id: record.id,
vector: {
[NEW_VECTOR]: {
text: ((record.payload?.text as string) ?? ""),
model: NEW_MODEL,
},
},
})),
});
reEmbedReachedEnd = reEmbedLastOffset == null;
}
// @block-end re-embed-existing
// @block-start search-with-old-vector
const oldVectorResults = await client.query(COLLECTION, {
query: {
text: "my query",
model: OLD_MODEL,
},
using: OLD_VECTOR,
limit: 10,
});
// @block-end search-with-old-vector
// @block-start search-with-new-vector
const newVectorResults = await client.query(COLLECTION, {
query: {
text: "my query",
model: NEW_MODEL,
},
using: NEW_VECTOR,
limit: 10,
});
// @block-end search-with-new-vector
// @block-start delete-old-named-vector
await client.deleteVectorName(COLLECTION, OLD_VECTOR);
// @block-end delete-old-named-vector