Model migration tutorial: use insert-only update mode and Cloud Inference (#2168)

* Switch to insert-only mode instead of conditional upserts

* Make code snippets testable; use Cloud Inference

* Use regular upserts instead of batch_update_points

* Add snippets for TS, Rust, Java, C#, and Go
This commit is contained in:
Abdon Pijpelink
2026-03-04 14:19:10 +01:00
committed by GitHub
parent 0725a70f05
commit 662e9aeaa3
49 changed files with 2257 additions and 105 deletions
@@ -0,0 +1,6 @@
```csharp
await client.CreateCollectionAsync(
collectionName: NEW_COLLECTION,
vectorsConfig: new VectorParams { Size = 512, Distance = Distance.Cosine }
);
```
@@ -0,0 +1,9 @@
```go
client.CreateCollection(context.Background(), &qdrant.CreateCollection{
CollectionName: NEW_COLLECTION,
VectorsConfig: qdrant.NewVectorsConfig(&qdrant.VectorParams{
Size: 512, // Size of the new embedding vectors
Distance: qdrant.Distance_Cosine,
}),
})
```
@@ -0,0 +1,7 @@
```java
client.createCollectionAsync(NEW_COLLECTION,
VectorParams.newBuilder()
.setSize(512) // Size of the new embedding vectors
.setDistance(Distance.Cosine) // Similarity function for the new model
.build()).get();
```
@@ -0,0 +1,11 @@
```python
client.create_collection(
collection_name=NEW_COLLECTION,
vectors_config=(
models.VectorParams(
size=512, # Size of the new embedding vectors
distance=models.Distance.COSINE # Similarity function for the new model
)
)
)
```
@@ -0,0 +1,8 @@
```rust
client
.create_collection(
CreateCollectionBuilder::new(new_collection)
.vectors_config(VectorParamsBuilder::new(512, Distance::Cosine)), // Size of the new embedding vectors
)
.await?;
```
@@ -0,0 +1,8 @@
```typescript
await client.createCollection(NEW_COLLECTION, {
vectors: {
size: 512, // Size of the new embedding vectors
distance: "Cosine", // Similarity function for the new model
},
});
```
@@ -0,0 +1,123 @@
```csharp
using Qdrant.Client;
using Qdrant.Client.Grpc;
await client.CreateCollectionAsync(
collectionName: NEW_COLLECTION,
vectorsConfig: new VectorParams { Size = 512, Distance = Distance.Cosine }
);
await client.UpsertAsync(
collectionName: OLD_COLLECTION,
points: new List<PointStruct>
{
new()
{
Id = 1,
Vectors = new Document
{
Text = "Example document",
Model = OLD_MODEL
},
Payload = { ["text"] = "Example document" }
}
}
);
await client.UpsertAsync(
collectionName: NEW_COLLECTION,
points: new List<PointStruct>
{
new()
{
Id = 1,
// Use the new embedding model to encode the document
Vectors = new Document
{
Text = "Example document",
Model = NEW_MODEL
},
Payload = { ["text"] = "Example document" }
}
}
);
PointId? lastOffset = null;
uint limit = 100; // Number of points to read in each batch
bool reachedEnd = false;
while (!reachedEnd)
{
// Get the next batch of points from the old collection
var scrollResult = await client.ScrollAsync(
collectionName: OLD_COLLECTION,
limit: limit,
offset: lastOffset,
// Include payloads in the response, as we need them to re-embed the vectors
payloadSelector: true,
// We don't need the old vectors, so let's save on the bandwidth
vectorsSelector: false
);
var records = scrollResult.Result;
lastOffset = scrollResult.NextPageOffset;
// Re-embed the points using the new model
var points = new List<PointStruct>();
foreach (var record in records)
{
var text = record.Payload.ContainsKey("text")
? record.Payload["text"].StringValue
: "";
points.Add(new PointStruct
{
// Keep the original ID to ensure consistency
Id = record.Id,
// Use the new embedding model to encode the text from the payload,
// assuming that was the original source of the embedding
Vectors = new Document
{
Text = text,
Model = NEW_MODEL
},
// Keep the original payload
Payload = { record.Payload }
});
}
// Upsert the re-embedded points into the new collection
await client.UpsertAsync(
new()
{
CollectionName = NEW_COLLECTION,
Points = { points },
// Only insert the point if a point with this ID does not already exist.
UpdateMode = UpdateMode.InsertOnly
}
);
// Check if we reached the end of the collection
reachedEnd = (lastOffset == null);
}
var results = await client.QueryAsync(
collectionName: OLD_COLLECTION,
query: new Document
{
Text = "my query",
Model = OLD_MODEL
},
limit: 10
);
results = await client.QueryAsync(
collectionName: NEW_COLLECTION,
query: new Document
{
Text = "my query",
Model = NEW_MODEL
},
limit: 10
);
```
@@ -0,0 +1,114 @@
```go
import (
"context"
"github.com/qdrant/go-client/qdrant"
)
client.CreateCollection(context.Background(), &qdrant.CreateCollection{
CollectionName: NEW_COLLECTION,
VectorsConfig: qdrant.NewVectorsConfig(&qdrant.VectorParams{
Size: 512, // Size of the new embedding vectors
Distance: qdrant.Distance_Cosine,
}),
})
client.Upsert(context.Background(), &qdrant.UpsertPoints{
CollectionName: OLD_COLLECTION,
Points: []*qdrant.PointStruct{
{
Id: qdrant.NewIDNum(1),
Vectors: qdrant.NewVectorsDocument(&qdrant.Document{
Text: "Example document",
Model: OLD_MODEL,
}),
Payload: qdrant.NewValueMap(map[string]any{"text": "Example document"}),
},
},
})
client.Upsert(context.Background(), &qdrant.UpsertPoints{
CollectionName: NEW_COLLECTION,
Points: []*qdrant.PointStruct{
{
Id: qdrant.NewIDNum(1),
// Use the new embedding model to encode the document
Vectors: qdrant.NewVectorsDocument(&qdrant.Document{
Text: "Example document",
Model: NEW_MODEL,
}),
Payload: qdrant.NewValueMap(map[string]any{"text": "Example document"}),
},
},
})
var lastOffset *qdrant.PointId
batchSize := uint32(100) // Number of points to read in each batch
reachedEnd := false
for !reachedEnd {
// Get the next batch of points from the old collection
scrollResult, err := client.Scroll(context.Background(), &qdrant.ScrollPoints{
CollectionName: OLD_COLLECTION,
Limit: qdrant.PtrOf(batchSize),
Offset: lastOffset,
// Include payloads in the response, as we need them to re-embed the vectors
WithPayload: qdrant.NewWithPayload(true),
// We don't need the old vectors, so let's save on the bandwidth
WithVectors: qdrant.NewWithVectors(false),
})
records := scrollResult
// Re-embed the points using the new model
points := make([]*qdrant.PointStruct, len(records))
for idx, record := range records {
text := ""
if val, ok := record.Payload["text"]; ok {
text = val.GetStringValue()
}
points[idx] = &qdrant.PointStruct{
// Keep the original ID to ensure consistency
Id: record.Id,
// Use the new embedding model to encode the text from the payload,
// assuming that was the original source of the embedding
Vectors: qdrant.NewVectorsDocument(&qdrant.Document{
Text: text,
Model: NEW_MODEL,
}),
// Keep the original payload
Payload: record.Payload,
}
}
// Upsert the re-embedded points into the new collection
client.Upsert(context.Background(), &qdrant.UpsertPoints{
CollectionName: NEW_COLLECTION,
Points: points,
// Only insert the point if a point with this ID does not already exist.
UpdateMode: qdrant.UpdateMode_InsertOnly.Enum(),
})
// Check if we reached the end of the collection
reachedEnd = (lastOffset == nil)
}
results, err := client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: OLD_COLLECTION,
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: "my query",
Model: OLD_MODEL,
}),
Limit: qdrant.PtrOf(uint64(10)),
})
results, err = client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: NEW_COLLECTION,
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: "my query",
Model: NEW_MODEL,
}),
Limit: qdrant.PtrOf(uint64(10)),
})
```
@@ -0,0 +1,144 @@
```java
import static io.qdrant.client.PointIdFactory.id;
import static io.qdrant.client.QueryFactory.nearest;
import static io.qdrant.client.ValueFactory.value;
import static io.qdrant.client.VectorFactory.vector;
import static io.qdrant.client.VectorsFactory.vectors;
import io.qdrant.client.WithPayloadSelectorFactory;
import io.qdrant.client.WithVectorsSelectorFactory;
import io.qdrant.client.QdrantClient;
import io.qdrant.client.QdrantGrpcClient;
import io.qdrant.client.grpc.Collections.Distance;
import io.qdrant.client.grpc.Collections.VectorParams;
import io.qdrant.client.grpc.JsonWithInt.Value;
import io.qdrant.client.grpc.Points.Document;
import io.qdrant.client.grpc.Points.PointStruct;
import io.qdrant.client.grpc.Points.QueryPoints;
import io.qdrant.client.grpc.Points.UpsertPoints;
import io.qdrant.client.grpc.Points.ScrollPoints;
import io.qdrant.client.grpc.Points.UpdateMode;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
client.createCollectionAsync(NEW_COLLECTION,
VectorParams.newBuilder()
.setSize(512) // Size of the new embedding vectors
.setDistance(Distance.Cosine) // Similarity function for the new model
.build()).get();
client.upsertAsync(OLD_COLLECTION, List.of(
PointStruct.newBuilder()
.setId(id(1))
.setVectors(
vectors(
vector(
Document.newBuilder()
.setText("Example document")
.setModel(OLD_MODEL)
.build())))
.putAllPayload(Map.of("text", value("Example document")))
.build())).get();
client.upsertAsync(NEW_COLLECTION, List.of(
PointStruct.newBuilder()
.setId(id(1))
// Use the new embedding model to encode the document
.setVectors(
vectors(
vector(
Document.newBuilder()
.setText("Example document")
.setModel(NEW_MODEL)
.build())))
.putAllPayload(Map.of("text", value("Example document")))
.build())).get();
int batchSize = 100; // Number of points to read in each batch
boolean reachedEnd = false;
// Get the next batch of points from the old collection
var scrollBuilder = ScrollPoints.newBuilder()
.setCollectionName(OLD_COLLECTION)
.setLimit(batchSize)
// Include payloads in the response, as we need them to re-embed the vectors
.setWithPayload(WithPayloadSelectorFactory.enable(true))
// We don't need the old vectors, so let's save on the bandwidth
.setWithVectors(WithVectorsSelectorFactory.enable(false));
while (!reachedEnd) {
var scrollResult = client.scrollAsync(scrollBuilder.build()).get();
var records = scrollResult.getResultList();
// Re-embed the points using the new model
List<PointStruct> points = new ArrayList<>();
for (var record : records) {
String text = record.getPayloadMap().containsKey("text")
? record.getPayloadMap().get("text").getStringValue()
: "";
points.add(
PointStruct.newBuilder()
// Keep the original ID to ensure consistency
.setId(record.getId())
// Use the new embedding model to encode the text from the payload,
// assuming that was the original source of the embedding
.setVectors(
vectors(
vector(
Document.newBuilder()
.setText(text)
.setModel(NEW_MODEL)
.build())))
// Keep the original payload
.putAllPayload(record.getPayloadMap())
.build());
}
// Upsert the re-embedded points into the new collection
client.upsertAsync(
UpsertPoints.newBuilder()
.setCollectionName(NEW_COLLECTION)
.addAllPoints(points)
// Only insert the point if a point with this ID does not already exist.
.setUpdateMode(UpdateMode.InsertOnly)
.build()).get();
// Check if we reached the end of the collection
if (scrollResult.hasNextPageOffset()) {
scrollBuilder.setOffset(scrollResult.getNextPageOffset());
} else {
reachedEnd = true;
}
}
QueryPoints oldRequest =
QueryPoints.newBuilder()
.setCollectionName(OLD_COLLECTION)
.setQuery(
nearest(
Document.newBuilder()
.setText("my query")
.setModel(OLD_MODEL)
.build()))
.setLimit(10)
.build();
var results = client.queryAsync(oldRequest).get();
QueryPoints newRequest =
QueryPoints.newBuilder()
.setCollectionName(NEW_COLLECTION)
.setQuery(
nearest(
Document.newBuilder()
.setText("my query")
.setModel(NEW_MODEL)
.build()))
.setLimit(10)
.build();
results = client.queryAsync(newRequest).get();
```
@@ -0,0 +1,60 @@
```csharp
PointId? lastOffset = null;
uint limit = 100; // Number of points to read in each batch
bool reachedEnd = false;
while (!reachedEnd)
{
// Get the next batch of points from the old collection
var scrollResult = await client.ScrollAsync(
collectionName: OLD_COLLECTION,
limit: limit,
offset: lastOffset,
// Include payloads in the response, as we need them to re-embed the vectors
payloadSelector: true,
// We don't need the old vectors, so let's save on the bandwidth
vectorsSelector: false
);
var records = scrollResult.Result;
lastOffset = scrollResult.NextPageOffset;
// Re-embed the points using the new model
var points = new List<PointStruct>();
foreach (var record in records)
{
var text = record.Payload.ContainsKey("text")
? record.Payload["text"].StringValue
: "";
points.Add(new PointStruct
{
// Keep the original ID to ensure consistency
Id = record.Id,
// Use the new embedding model to encode the text from the payload,
// assuming that was the original source of the embedding
Vectors = new Document
{
Text = text,
Model = NEW_MODEL
},
// Keep the original payload
Payload = { record.Payload }
});
}
// Upsert the re-embedded points into the new collection
await client.UpsertAsync(
new()
{
CollectionName = NEW_COLLECTION,
Points = { points },
// Only insert the point if a point with this ID does not already exist.
UpdateMode = UpdateMode.InsertOnly
}
);
// Check if we reached the end of the collection
reachedEnd = (lastOffset == null);
}
```
@@ -0,0 +1,53 @@
```go
var lastOffset *qdrant.PointId
batchSize := uint32(100) // Number of points to read in each batch
reachedEnd := false
for !reachedEnd {
// Get the next batch of points from the old collection
scrollResult, err := client.Scroll(context.Background(), &qdrant.ScrollPoints{
CollectionName: OLD_COLLECTION,
Limit: qdrant.PtrOf(batchSize),
Offset: lastOffset,
// Include payloads in the response, as we need them to re-embed the vectors
WithPayload: qdrant.NewWithPayload(true),
// We don't need the old vectors, so let's save on the bandwidth
WithVectors: qdrant.NewWithVectors(false),
})
records := scrollResult
// Re-embed the points using the new model
points := make([]*qdrant.PointStruct, len(records))
for idx, record := range records {
text := ""
if val, ok := record.Payload["text"]; ok {
text = val.GetStringValue()
}
points[idx] = &qdrant.PointStruct{
// Keep the original ID to ensure consistency
Id: record.Id,
// Use the new embedding model to encode the text from the payload,
// assuming that was the original source of the embedding
Vectors: qdrant.NewVectorsDocument(&qdrant.Document{
Text: text,
Model: NEW_MODEL,
}),
// Keep the original payload
Payload: record.Payload,
}
}
// Upsert the re-embedded points into the new collection
client.Upsert(context.Background(), &qdrant.UpsertPoints{
CollectionName: NEW_COLLECTION,
Points: points,
// Only insert the point if a point with this ID does not already exist.
UpdateMode: qdrant.UpdateMode_InsertOnly.Enum(),
})
// Check if we reached the end of the collection
reachedEnd = (lastOffset == nil)
}
```
@@ -0,0 +1,60 @@
```java
int batchSize = 100; // Number of points to read in each batch
boolean reachedEnd = false;
// Get the next batch of points from the old collection
var scrollBuilder = ScrollPoints.newBuilder()
.setCollectionName(OLD_COLLECTION)
.setLimit(batchSize)
// Include payloads in the response, as we need them to re-embed the vectors
.setWithPayload(WithPayloadSelectorFactory.enable(true))
// We don't need the old vectors, so let's save on the bandwidth
.setWithVectors(WithVectorsSelectorFactory.enable(false));
while (!reachedEnd) {
var scrollResult = client.scrollAsync(scrollBuilder.build()).get();
var records = scrollResult.getResultList();
// Re-embed the points using the new model
List<PointStruct> points = new ArrayList<>();
for (var record : records) {
String text = record.getPayloadMap().containsKey("text")
? record.getPayloadMap().get("text").getStringValue()
: "";
points.add(
PointStruct.newBuilder()
// Keep the original ID to ensure consistency
.setId(record.getId())
// Use the new embedding model to encode the text from the payload,
// assuming that was the original source of the embedding
.setVectors(
vectors(
vector(
Document.newBuilder()
.setText(text)
.setModel(NEW_MODEL)
.build())))
// Keep the original payload
.putAllPayload(record.getPayloadMap())
.build());
}
// Upsert the re-embedded points into the new collection
client.upsertAsync(
UpsertPoints.newBuilder()
.setCollectionName(NEW_COLLECTION)
.addAllPoints(points)
// Only insert the point if a point with this ID does not already exist.
.setUpdateMode(UpdateMode.InsertOnly)
.build()).get();
// Check if we reached the end of the collection
if (scrollResult.hasNextPageOffset()) {
scrollBuilder.setOffset(scrollResult.getNextPageOffset());
} else {
reachedEnd = true;
}
}
```
@@ -0,0 +1,45 @@
```python
last_offset = None
batch_size = 100 # Number of points to read in each batch
reached_end = False
while not reached_end:
# Get the next batch of points from the old collection
records, last_offset = client.scroll(
collection_name=OLD_COLLECTION,
limit=batch_size,
offset=last_offset,
# Include payloads in the response, as we need them to re-embed the vectors
with_payload=True,
# We don't need the old vectors, so let's save on the bandwidth
with_vectors=False,
)
# Re-embed the points using the new model
points = [
models.PointStruct(
# Keep the original ID to ensure consistency
id=record.id,
# Use the new embedding model to encode the text from the payload,
# assuming that was the original source of the embedding
vector=models.Document(
text=(record.payload or {}).get("text", ""),
model=NEW_MODEL,
),
# Keep the original payload
payload=record.payload
)
for record in records
]
# Upsert the re-embedded points into the new collection
client.upsert(
collection_name=NEW_COLLECTION,
points=points,
# Only insert the point if a point with this ID does not already exist.
update_mode=models.UpdateMode.INSERT_ONLY
)
# Check if we reached the end of the collection
reached_end = (last_offset == None)
```
@@ -0,0 +1,58 @@
```rust
let mut last_offset = None;
let batch_size = 100; // Number of points to read in each batch
loop {
// Get the next batch of points from the old collection
let mut scroll_builder = ScrollPointsBuilder::new(old_collection)
.limit(batch_size)
// Include payloads in the response, as we need them to re-embed the vectors
.with_payload(true)
// We don't need the old vectors, so let's save on the bandwidth
.with_vectors(false);
if let Some(offset) = last_offset {
scroll_builder = scroll_builder.offset(offset);
}
let scroll_result = client.scroll(scroll_builder).await?;
let records = scroll_result.result;
last_offset = scroll_result.next_page_offset;
// Re-embed the points using the new model
let points: Vec<PointStruct> = records
.iter()
.map(|record| {
PointStruct::new(
// Keep the original ID to ensure consistency
record.id.clone().unwrap(),
// Use the new embedding model to encode the text from the payload,
// assuming that was the original source of the embedding
Document::new(
record.payload.get("text")
.and_then(|v| v.as_str())
.map_or("", |v| v),
new_model,
),
// Keep the original payload
record.payload.clone(),
)
})
.collect();
// Upsert the re-embedded points into the new collection
client
.upsert_points(
// Only insert the point if a point with this ID does not already exist.
UpsertPointsBuilder::new(new_collection, points)
.update_mode(UpdateMode::InsertOnly),
)
.await?;
// Check if we reached the end of the collection
if last_offset.is_none() {
break;
}
}
```
@@ -0,0 +1,44 @@
```typescript
let lastOffset: number | string | undefined = undefined;
const batchSize = 100; // Number of points to read in each batch
let reachedEnd = false;
while (!reachedEnd) {
// Get the next batch of points from the old collection
const scrollResult = await client.scroll(OLD_COLLECTION, {
limit: batchSize,
offset: lastOffset,
// Include payloads in the response, as we need them to re-embed the vectors
with_payload: true,
// We don't need the old vectors, so let's save on the bandwidth
with_vector: false,
});
const records = scrollResult.points;
lastOffset = scrollResult.next_page_offset as number | string | undefined;
// Re-embed the points using the new model
const points = records.map((record) => ({
// Keep the original ID to ensure consistency
id: record.id,
// Use the new embedding model to encode the text from the payload,
// assuming that was the original source of the embedding
vector: {
text: ((record.payload?.text as string) ?? ""),
model: NEW_MODEL,
},
// Keep the original payload
payload: record.payload,
}));
// Upsert the re-embedded points into the new collection
await client.upsert(NEW_COLLECTION, {
points,
// Only insert the point if a point with this ID does not already exist.
update_mode: "insert_only" as const,
});
// Check if we reached the end of the collection
reachedEnd = lastOffset == null;
}
```
@@ -0,0 +1,98 @@
```python
from qdrant_client import QdrantClient, models
client.create_collection(
collection_name=NEW_COLLECTION,
vectors_config=(
models.VectorParams(
size=512, # Size of the new embedding vectors
distance=models.Distance.COSINE # Similarity function for the new model
)
)
)
client.upsert(
collection_name=OLD_COLLECTION,
points=[
models.PointStruct(
id=1,
vector=models.Document(
text="Example document",
model=OLD_MODEL,
),
payload={"text": "Example document"}
)
]
)
client.upsert(
collection_name=NEW_COLLECTION,
points=[
models.PointStruct(
id=1,
# Use the new embedding model to encode the document
vector=models.Document(
text="Example document",
model=NEW_MODEL,
),
payload={"text": "Example document"}
)
]
)
last_offset = None
batch_size = 100 # Number of points to read in each batch
reached_end = False
while not reached_end:
# Get the next batch of points from the old collection
records, last_offset = client.scroll(
collection_name=OLD_COLLECTION,
limit=batch_size,
offset=last_offset,
# Include payloads in the response, as we need them to re-embed the vectors
with_payload=True,
# We don't need the old vectors, so let's save on the bandwidth
with_vectors=False,
)
# Re-embed the points using the new model
points = [
models.PointStruct(
# Keep the original ID to ensure consistency
id=record.id,
# Use the new embedding model to encode the text from the payload,
# assuming that was the original source of the embedding
vector=models.Document(
text=(record.payload or {}).get("text", ""),
model=NEW_MODEL,
),
# Keep the original payload
payload=record.payload
)
for record in records
]
# Upsert the re-embedded points into the new collection
client.upsert(
collection_name=NEW_COLLECTION,
points=points,
# Only insert the point if a point with this ID does not already exist.
update_mode=models.UpdateMode.INSERT_ONLY
)
# Check if we reached the end of the collection
reached_end = (last_offset == None)
results = client.query_points(
collection_name=OLD_COLLECTION,
query=models.Document(text="my query", model=OLD_MODEL),
limit=10,
)
results = client.query_points(
collection_name=NEW_COLLECTION,
query=models.Document(text="my query", model=NEW_MODEL),
limit=10,
)
```
@@ -0,0 +1,110 @@
```rust
use qdrant_client::qdrant::{
CreateCollectionBuilder, Distance, Document, PointStruct, Query, QueryPointsBuilder,
ScrollPointsBuilder, UpdateMode, UpsertPointsBuilder, VectorParamsBuilder,
};
use qdrant_client::Qdrant;
client
.create_collection(
CreateCollectionBuilder::new(new_collection)
.vectors_config(VectorParamsBuilder::new(512, Distance::Cosine)), // Size of the new embedding vectors
)
.await?;
client
.upsert_points(UpsertPointsBuilder::new(
old_collection,
vec![PointStruct::new(
1,
Document::new("Example document", old_model),
[("text", "Example document".into())],
)],
))
.await?;
client
.upsert_points(UpsertPointsBuilder::new(
new_collection,
vec![PointStruct::new(
1,
// Use the new embedding model to encode the document
Document::new("Example document", new_model),
[("text", "Example document".into())],
)],
))
.await?;
let mut last_offset = None;
let batch_size = 100; // Number of points to read in each batch
loop {
// Get the next batch of points from the old collection
let mut scroll_builder = ScrollPointsBuilder::new(old_collection)
.limit(batch_size)
// Include payloads in the response, as we need them to re-embed the vectors
.with_payload(true)
// We don't need the old vectors, so let's save on the bandwidth
.with_vectors(false);
if let Some(offset) = last_offset {
scroll_builder = scroll_builder.offset(offset);
}
let scroll_result = client.scroll(scroll_builder).await?;
let records = scroll_result.result;
last_offset = scroll_result.next_page_offset;
// Re-embed the points using the new model
let points: Vec<PointStruct> = records
.iter()
.map(|record| {
PointStruct::new(
// Keep the original ID to ensure consistency
record.id.clone().unwrap(),
// Use the new embedding model to encode the text from the payload,
// assuming that was the original source of the embedding
Document::new(
record.payload.get("text")
.and_then(|v| v.as_str())
.map_or("", |v| v),
new_model,
),
// Keep the original payload
record.payload.clone(),
)
})
.collect();
// Upsert the re-embedded points into the new collection
client
.upsert_points(
// Only insert the point if a point with this ID does not already exist.
UpsertPointsBuilder::new(new_collection, points)
.update_mode(UpdateMode::InsertOnly),
)
.await?;
// Check if we reached the end of the collection
if last_offset.is_none() {
break;
}
}
let results = client
.query(
QueryPointsBuilder::new(old_collection)
.query(Query::new_nearest(Document::new("my query", old_model)))
.limit(10),
)
.await?;
let results = client
.query(
QueryPointsBuilder::new(new_collection)
.query(Query::new_nearest(Document::new("my query", new_model)))
.limit(10),
)
.await?;
```
@@ -0,0 +1,11 @@
```csharp
results = await client.QueryAsync(
collectionName: NEW_COLLECTION,
query: new Document
{
Text = "my query",
Model = NEW_MODEL
},
limit: 10
);
```
@@ -0,0 +1,10 @@
```go
results, err = client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: NEW_COLLECTION,
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: "my query",
Model: NEW_MODEL,
}),
Limit: qdrant.PtrOf(uint64(10)),
})
```
@@ -0,0 +1,15 @@
```java
QueryPoints newRequest =
QueryPoints.newBuilder()
.setCollectionName(NEW_COLLECTION)
.setQuery(
nearest(
Document.newBuilder()
.setText("my query")
.setModel(NEW_MODEL)
.build()))
.setLimit(10)
.build();
results = client.queryAsync(newRequest).get();
```
@@ -0,0 +1,7 @@
```python
results = client.query_points(
collection_name=NEW_COLLECTION,
query=models.Document(text="my query", model=NEW_MODEL),
limit=10,
)
```
@@ -0,0 +1,9 @@
```rust
let results = client
.query(
QueryPointsBuilder::new(new_collection)
.query(Query::new_nearest(Document::new("my query", new_model)))
.limit(10),
)
.await?;
```
@@ -0,0 +1,9 @@
```typescript
const resultsNew = await client.query(NEW_COLLECTION, {
query: {
text: "my query",
model: NEW_MODEL,
},
limit: 10,
});
```
@@ -0,0 +1,11 @@
```csharp
var results = await client.QueryAsync(
collectionName: OLD_COLLECTION,
query: new Document
{
Text = "my query",
Model = OLD_MODEL
},
limit: 10
);
```
@@ -0,0 +1,10 @@
```go
results, err := client.Query(context.Background(), &qdrant.QueryPoints{
CollectionName: OLD_COLLECTION,
Query: qdrant.NewQueryDocument(&qdrant.Document{
Text: "my query",
Model: OLD_MODEL,
}),
Limit: qdrant.PtrOf(uint64(10)),
})
```
@@ -0,0 +1,15 @@
```java
QueryPoints oldRequest =
QueryPoints.newBuilder()
.setCollectionName(OLD_COLLECTION)
.setQuery(
nearest(
Document.newBuilder()
.setText("my query")
.setModel(OLD_MODEL)
.build()))
.setLimit(10)
.build();
var results = client.queryAsync(oldRequest).get();
```
@@ -0,0 +1,7 @@
```python
results = client.query_points(
collection_name=OLD_COLLECTION,
query=models.Document(text="my query", model=OLD_MODEL),
limit=10,
)
```
@@ -0,0 +1,9 @@
```rust
let results = client
.query(
QueryPointsBuilder::new(old_collection)
.query(Query::new_nearest(Document::new("my query", old_model)))
.limit(10),
)
.await?;
```
@@ -0,0 +1,9 @@
```typescript
const results = await client.query(OLD_COLLECTION, {
query: {
text: "my query",
model: OLD_MODEL,
},
limit: 10,
});
```
@@ -0,0 +1,96 @@
```typescript
import { QdrantClient } from "@qdrant/js-client-rest";
await client.createCollection(NEW_COLLECTION, {
vectors: {
size: 512, // Size of the new embedding vectors
distance: "Cosine", // Similarity function for the new model
},
});
await client.upsert(OLD_COLLECTION, {
points: [
{
id: 1,
vector: {
text: "Example document",
model: OLD_MODEL,
},
payload: { text: "Example document" },
},
],
});
await client.upsert(NEW_COLLECTION, {
points: [
{
id: 1,
// Use the new embedding model to encode the document
vector: {
text: "Example document",
model: NEW_MODEL,
},
payload: { text: "Example document" },
},
],
});
let lastOffset: number | string | undefined = undefined;
const batchSize = 100; // Number of points to read in each batch
let reachedEnd = false;
while (!reachedEnd) {
// Get the next batch of points from the old collection
const scrollResult = await client.scroll(OLD_COLLECTION, {
limit: batchSize,
offset: lastOffset,
// Include payloads in the response, as we need them to re-embed the vectors
with_payload: true,
// We don't need the old vectors, so let's save on the bandwidth
with_vector: false,
});
const records = scrollResult.points;
lastOffset = scrollResult.next_page_offset as number | string | undefined;
// Re-embed the points using the new model
const points = records.map((record) => ({
// Keep the original ID to ensure consistency
id: record.id,
// Use the new embedding model to encode the text from the payload,
// assuming that was the original source of the embedding
vector: {
text: ((record.payload?.text as string) ?? ""),
model: NEW_MODEL,
},
// Keep the original payload
payload: record.payload,
}));
// Upsert the re-embedded points into the new collection
await client.upsert(NEW_COLLECTION, {
points,
// Only insert the point if a point with this ID does not already exist.
update_mode: "insert_only" as const,
});
// Check if we reached the end of the collection
reachedEnd = lastOffset == null;
}
const results = await client.query(OLD_COLLECTION, {
query: {
text: "my query",
model: OLD_MODEL,
},
limit: 10,
});
const resultsNew = await client.query(NEW_COLLECTION, {
query: {
text: "my query",
model: NEW_MODEL,
},
limit: 10,
});
```
@@ -0,0 +1,19 @@
```csharp
await client.UpsertAsync(
collectionName: NEW_COLLECTION,
points: new List<PointStruct>
{
new()
{
Id = 1,
// Use the new embedding model to encode the document
Vectors = new Document
{
Text = "Example document",
Model = NEW_MODEL
},
Payload = { ["text"] = "Example document" }
}
}
);
```
@@ -0,0 +1,16 @@
```go
client.Upsert(context.Background(), &qdrant.UpsertPoints{
CollectionName: NEW_COLLECTION,
Points: []*qdrant.PointStruct{
{
Id: qdrant.NewIDNum(1),
// Use the new embedding model to encode the document
Vectors: qdrant.NewVectorsDocument(&qdrant.Document{
Text: "Example document",
Model: NEW_MODEL,
}),
Payload: qdrant.NewValueMap(map[string]any{"text": "Example document"}),
},
},
})
```
@@ -0,0 +1,15 @@
```java
client.upsertAsync(NEW_COLLECTION, List.of(
PointStruct.newBuilder()
.setId(id(1))
// Use the new embedding model to encode the document
.setVectors(
vectors(
vector(
Document.newBuilder()
.setText("Example document")
.setModel(NEW_MODEL)
.build())))
.putAllPayload(Map.of("text", value("Example document")))
.build())).get();
```
@@ -0,0 +1,16 @@
```python
client.upsert(
collection_name=NEW_COLLECTION,
points=[
models.PointStruct(
id=1,
# Use the new embedding model to encode the document
vector=models.Document(
text="Example document",
model=NEW_MODEL,
),
payload={"text": "Example document"}
)
]
)
```
@@ -0,0 +1,13 @@
```rust
client
.upsert_points(UpsertPointsBuilder::new(
new_collection,
vec![PointStruct::new(
1,
// Use the new embedding model to encode the document
Document::new("Example document", new_model),
[("text", "Example document".into())],
)],
))
.await?;
```
@@ -0,0 +1,15 @@
```typescript
await client.upsert(NEW_COLLECTION, {
points: [
{
id: 1,
// Use the new embedding model to encode the document
vector: {
text: "Example document",
model: NEW_MODEL,
},
payload: { text: "Example document" },
},
],
});
```
@@ -0,0 +1,18 @@
```csharp
await client.UpsertAsync(
collectionName: OLD_COLLECTION,
points: new List<PointStruct>
{
new()
{
Id = 1,
Vectors = new Document
{
Text = "Example document",
Model = OLD_MODEL
},
Payload = { ["text"] = "Example document" }
}
}
);
```
@@ -0,0 +1,15 @@
```go
client.Upsert(context.Background(), &qdrant.UpsertPoints{
CollectionName: OLD_COLLECTION,
Points: []*qdrant.PointStruct{
{
Id: qdrant.NewIDNum(1),
Vectors: qdrant.NewVectorsDocument(&qdrant.Document{
Text: "Example document",
Model: OLD_MODEL,
}),
Payload: qdrant.NewValueMap(map[string]any{"text": "Example document"}),
},
},
})
```
@@ -0,0 +1,14 @@
```java
client.upsertAsync(OLD_COLLECTION, List.of(
PointStruct.newBuilder()
.setId(id(1))
.setVectors(
vectors(
vector(
Document.newBuilder()
.setText("Example document")
.setModel(OLD_MODEL)
.build())))
.putAllPayload(Map.of("text", value("Example document")))
.build())).get();
```
@@ -0,0 +1,15 @@
```python
client.upsert(
collection_name=OLD_COLLECTION,
points=[
models.PointStruct(
id=1,
vector=models.Document(
text="Example document",
model=OLD_MODEL,
),
payload={"text": "Example document"}
)
]
)
```
@@ -0,0 +1,12 @@
```rust
client
.upsert_points(UpsertPointsBuilder::new(
old_collection,
vec![PointStruct::new(
1,
Document::new("Example document", old_model),
[("text", "Example document".into())],
)],
))
.await?;
```
@@ -0,0 +1,14 @@
```typescript
await client.upsert(OLD_COLLECTION, {
points: [
{
id: 1,
vector: {
text: "Example document",
model: OLD_MODEL,
},
payload: { text: "Example document" },
},
],
});
```