diff --git a/qdrant-landing/content/blog/qdrant-1.18.x.md b/qdrant-landing/content/blog/qdrant-1.18.x.md index 22947be5d..d89ee050d 100644 --- a/qdrant-landing/content/blog/qdrant-1.18.x.md +++ b/qdrant-landing/content/blog/qdrant-1.18.x.md @@ -101,7 +101,7 @@ The same data is available [through Qdrant's API](/documentation/ops-monitoring/ When a collection's vector schema needed to change, for example, when adding support for a new embedding model or retiring an old one, there was no way to update it in place. The only option was to recreate the collection from scratch and re-ingest all points. -Version 1.18 makes it possible to [add and remove named vectors](/documentation/manage-data/collections/#update-vectors) to an existing collection's schema without having to recreate it. This makes embedding model migration considerably easier. You can now add a new vector field, populate it in the background, and remove the old one when you're ready. +Version 1.18 makes it possible to [add and remove named vectors](/documentation/manage-data/collections/#update-vector-schema) to an existing collection's schema without having to recreate it. This makes embedding model migration considerably easier. You can now add a new vector field, populate it in the background, and remove the old one when you're ready. ## Audit Logging Improvements diff --git a/qdrant-landing/content/documentation/faq/qdrant-fundamentals.md b/qdrant-landing/content/documentation/faq/qdrant-fundamentals.md index 5516997d2..99e6daaea 100644 --- a/qdrant-landing/content/documentation/faq/qdrant-fundamentals.md +++ b/qdrant-landing/content/documentation/faq/qdrant-fundamentals.md @@ -220,14 +220,18 @@ See also: [Named Vectors](/documentation/manage-data/vectors/#named-vectors) ### Can I switch to a different embedding model without recreating my collection? -The recommended pattern is an alias-based swap: +Yes, if you're using named vectors: + +1. [Add a new named vector](/documentation/manage-data/collections/#update-vector-schema) for the new model +2. Re-embed points in the background +3. Remove the old named vector when you're ready. + +If you don't use named vectors, you need to create a new collection. Use an alias-based swap for a zero-downtime migration: 1. Create a new collection and ingest your data re-embedded with the new model. 2. Once indexed, atomically update the [collection alias](/documentation/manage-data/collections/#collection-aliases) to point to the new collection. 3. Delete the old collection when you're confident the migration is stable. -This gives you zero-downtime migrations and a rollback path. - See also: [Migrate to a New Embedding Model](/documentation/tutorials-operations/embedding-model-migration/) ### Why is my collection in "grey" status? diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/csharp.cs b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/csharp.cs index 3b4f075ba..4dbf3b966 100644 --- a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/csharp.cs +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/csharp.cs @@ -18,6 +18,10 @@ public class Snippet string OLD_MODEL = "sentence-transformers/all-minilm-l6-v2"; string NEW_MODEL = "qdrant/clip-vit-b-32-text"; + + string COLLECTION = "my_collection"; + string OLD_VECTOR = "old-model"; + string NEW_VECTOR = "new-model"; // @hide-end // @block-start create-new-collection @@ -150,5 +154,101 @@ public class Snippet limit: 10 ); // @block-end search-new-collection + + // @block-start add-named-vector + await client.CreateVectorNameAsync(new() + { + CollectionName = COLLECTION, + VectorName = NEW_VECTOR, + DenseConfig = new() { Size = 512, Distance = Distance.Cosine } + }); + // @block-end add-named-vector + + // @block-start upsert-both-vectors + await client.UpsertAsync( + collectionName: COLLECTION, + points: new List + { + new() + { + Id = 1, + Vectors = new Dictionary + { + [OLD_VECTOR] = new Document { Text = "Example document", Model = OLD_MODEL }, + [NEW_VECTOR] = new Document { Text = "Example document", Model = NEW_MODEL }, + }, + Payload = { ["text"] = "Example document" } + } + } + ); + // @block-end upsert-both-vectors + + // @block-start re-embed-existing + PointId? reEmbedLastOffset = null; + uint reEmbedBatchSize = 100; + bool reEmbedReachedEnd = false; + + while (!reEmbedReachedEnd) + { + var reEmbedScrollResult = await client.ScrollAsync( + collectionName: COLLECTION, + limit: reEmbedBatchSize, + offset: reEmbedLastOffset, + payloadSelector: true, + vectorsSelector: false + ); + + var reEmbedRecords = reEmbedScrollResult.Result; + reEmbedLastOffset = reEmbedScrollResult.NextPageOffset; + + var pointVectors = new List(); + foreach (var record in reEmbedRecords) + { + var text = record.Payload.ContainsKey("text") + ? record.Payload["text"].StringValue + : ""; + + // Update only the new vector on each point; the old vector and payload are untouched + pointVectors.Add(new PointVectors + { + Id = record.Id, + Vectors = new Dictionary + { + [NEW_VECTOR] = new Document { Text = text, Model = NEW_MODEL } + } + }); + } + + await client.UpdateVectorsAsync(collectionName: COLLECTION, points: pointVectors); + + reEmbedReachedEnd = (reEmbedLastOffset == null); + } + // @block-end re-embed-existing + + // @block-start search-with-old-vector + var oldVectorResults = await client.QueryAsync( + collectionName: COLLECTION, + query: new Document { Text = "my query", Model = OLD_MODEL }, + usingVector: OLD_VECTOR, + limit: 10 + ); + // @block-end search-with-old-vector + + // @block-start search-with-new-vector + var newVectorResults = await client.QueryAsync( + collectionName: COLLECTION, + query: new Document { Text = "my query", Model = NEW_MODEL }, + usingVector: NEW_VECTOR, + limit: 10 + ); + // @block-end search-with-new-vector + + // @block-start delete-old-named-vector + await client.DeleteVectorNameAsync(new() + { + CollectionName = COLLECTION, + VectorName = OLD_VECTOR + }); + // @block-end delete-old-named-vector } } diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/add-named-vector/csharp.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/add-named-vector/csharp.md new file mode 100644 index 000000000..12302944e --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/add-named-vector/csharp.md @@ -0,0 +1,8 @@ +```csharp +await client.CreateVectorNameAsync(new() +{ + CollectionName = COLLECTION, + VectorName = NEW_VECTOR, + DenseConfig = new() { Size = 512, Distance = Distance.Cosine } +}); +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/add-named-vector/go.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/add-named-vector/go.md new file mode 100644 index 000000000..cd5ea25c1 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/add-named-vector/go.md @@ -0,0 +1,12 @@ +```go +client.CreateVectorName(context.Background(), &qdrant.CreateVectorNameRequest{ + CollectionName: COLLECTION, + VectorName: NEW_VECTOR, + VectorConfig: &qdrant.CreateVectorNameRequest_DenseConfig{ + DenseConfig: &qdrant.DenseVectorCreationConfig{ + Size: 512, // Size of the new embedding vectors + Distance: qdrant.Distance_Cosine, + }, + }, +}) +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/add-named-vector/java.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/add-named-vector/java.md new file mode 100644 index 000000000..bb7a8568a --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/add-named-vector/java.md @@ -0,0 +1,14 @@ +```java +client + .createVectorNameAsync( + CreateVectorNameRequest.newBuilder() + .setCollectionName(COLLECTION) + .setVectorName(NEW_VECTOR) + .setDenseConfig( + DenseVectorCreationConfig.newBuilder() + .setSize(512) // Size of the new embedding vectors + .setDistance(Distance.Cosine) // Similarity function for the new model + .build()) + .build()) + .get(); +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/add-named-vector/python.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/add-named-vector/python.md new file mode 100644 index 000000000..46857124a --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/add-named-vector/python.md @@ -0,0 +1,12 @@ +```python +client.create_vector_name( + collection_name=COLLECTION, + vector_name=NEW_VECTOR, + vector_name_config=models.DenseVectorNameConfig( + dense=models.DenseVectorConfig( + size=512, # Size of the new embedding vectors + distance=models.Distance.COSINE # Similarity function for the new model + ) + ), +) +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/add-named-vector/rust.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/add-named-vector/rust.md new file mode 100644 index 000000000..6b7f8a20e --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/add-named-vector/rust.md @@ -0,0 +1,11 @@ +```rust +client + .create_vector_name( + CreateVectorNameRequestBuilder::new( + collection, + new_vector, + DenseVectorCreationConfigBuilder::new(512, Distance::Cosine), // Size of the new embedding vectors + ), + ) + .await?; +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/add-named-vector/typescript.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/add-named-vector/typescript.md new file mode 100644 index 000000000..70a65e329 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/add-named-vector/typescript.md @@ -0,0 +1,8 @@ +```typescript +await client.createVectorName(COLLECTION, NEW_VECTOR, { + dense: { + size: 512, // Size of the new embedding vectors + distance: "Cosine", // Similarity function for the new model + }, +}); +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/csharp.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/csharp.md index 6db698af5..4257bdf79 100644 --- a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/csharp.md +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/csharp.md @@ -120,4 +120,88 @@ results = await client.QueryAsync( }, limit: 10 ); + +await client.CreateVectorNameAsync(new() +{ + CollectionName = COLLECTION, + VectorName = NEW_VECTOR, + DenseConfig = new() { Size = 512, Distance = Distance.Cosine } +}); + +await client.UpsertAsync( + collectionName: COLLECTION, + points: new List + { + new() + { + Id = 1, + Vectors = new Dictionary + { + [OLD_VECTOR] = new Document { Text = "Example document", Model = OLD_MODEL }, + [NEW_VECTOR] = new Document { Text = "Example document", Model = NEW_MODEL }, + }, + Payload = { ["text"] = "Example document" } + } + } +); + +PointId? reEmbedLastOffset = null; +uint reEmbedBatchSize = 100; +bool reEmbedReachedEnd = false; + +while (!reEmbedReachedEnd) +{ + var reEmbedScrollResult = await client.ScrollAsync( + collectionName: COLLECTION, + limit: reEmbedBatchSize, + offset: reEmbedLastOffset, + payloadSelector: true, + vectorsSelector: false + ); + + var reEmbedRecords = reEmbedScrollResult.Result; + reEmbedLastOffset = reEmbedScrollResult.NextPageOffset; + + var pointVectors = new List(); + foreach (var record in reEmbedRecords) + { + var text = record.Payload.ContainsKey("text") + ? record.Payload["text"].StringValue + : ""; + + // Update only the new vector on each point; the old vector and payload are untouched + pointVectors.Add(new PointVectors + { + Id = record.Id, + Vectors = new Dictionary + { + [NEW_VECTOR] = new Document { Text = text, Model = NEW_MODEL } + } + }); + } + + await client.UpdateVectorsAsync(collectionName: COLLECTION, points: pointVectors); + + reEmbedReachedEnd = (reEmbedLastOffset == null); +} + +var oldVectorResults = await client.QueryAsync( + collectionName: COLLECTION, + query: new Document { Text = "my query", Model = OLD_MODEL }, + usingVector: OLD_VECTOR, + limit: 10 +); + +var newVectorResults = await client.QueryAsync( + collectionName: COLLECTION, + query: new Document { Text = "my query", Model = NEW_MODEL }, + usingVector: NEW_VECTOR, + limit: 10 +); + +await client.DeleteVectorNameAsync(new() +{ + CollectionName = COLLECTION, + VectorName = OLD_VECTOR +}); ``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/delete-old-named-vector/csharp.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/delete-old-named-vector/csharp.md new file mode 100644 index 000000000..271893cb2 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/delete-old-named-vector/csharp.md @@ -0,0 +1,7 @@ +```csharp +await client.DeleteVectorNameAsync(new() +{ + CollectionName = COLLECTION, + VectorName = OLD_VECTOR +}); +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/delete-old-named-vector/go.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/delete-old-named-vector/go.md new file mode 100644 index 000000000..3e1cc2691 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/delete-old-named-vector/go.md @@ -0,0 +1,6 @@ +```go +client.DeleteVectorName(context.Background(), &qdrant.DeleteVectorNameRequest{ + CollectionName: COLLECTION, + VectorName: OLD_VECTOR, +}) +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/delete-old-named-vector/java.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/delete-old-named-vector/java.md new file mode 100644 index 000000000..412e37e57 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/delete-old-named-vector/java.md @@ -0,0 +1,9 @@ +```java +client + .deleteVectorNameAsync( + DeleteVectorNameRequest.newBuilder() + .setCollectionName(COLLECTION) + .setVectorName(OLD_VECTOR) + .build()) + .get(); +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/delete-old-named-vector/python.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/delete-old-named-vector/python.md new file mode 100644 index 000000000..35fea9696 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/delete-old-named-vector/python.md @@ -0,0 +1,6 @@ +```python +client.delete_vector_name( + collection_name=COLLECTION, + vector_name=OLD_VECTOR, +) +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/delete-old-named-vector/rust.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/delete-old-named-vector/rust.md new file mode 100644 index 000000000..e6191998c --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/delete-old-named-vector/rust.md @@ -0,0 +1,8 @@ +```rust +client + .delete_vector_name(DeleteVectorNameRequestBuilder::new( + collection, + old_vector, + )) + .await?; +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/delete-old-named-vector/typescript.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/delete-old-named-vector/typescript.md new file mode 100644 index 000000000..d29d892a9 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/delete-old-named-vector/typescript.md @@ -0,0 +1,3 @@ +```typescript +await client.deleteVectorName(COLLECTION, OLD_VECTOR); +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/go.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/go.md index ac4910a7d..3fdc1a2eb 100644 --- a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/go.md +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/go.md @@ -111,4 +111,102 @@ results, err = client.Query(context.Background(), &qdrant.QueryPoints{ }), Limit: qdrant.PtrOf(uint64(10)), }) + +client.CreateVectorName(context.Background(), &qdrant.CreateVectorNameRequest{ + CollectionName: COLLECTION, + VectorName: NEW_VECTOR, + VectorConfig: &qdrant.CreateVectorNameRequest_DenseConfig{ + DenseConfig: &qdrant.DenseVectorCreationConfig{ + Size: 512, // Size of the new embedding vectors + Distance: qdrant.Distance_Cosine, + }, + }, +}) + +client.Upsert(context.Background(), &qdrant.UpsertPoints{ + CollectionName: COLLECTION, + Points: []*qdrant.PointStruct{ + { + Id: qdrant.NewIDNum(1), + Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{ + OLD_VECTOR: qdrant.NewVectorDocument(&qdrant.Document{ + Text: "Example document", + Model: OLD_MODEL, + }), + NEW_VECTOR: qdrant.NewVectorDocument(&qdrant.Document{ + Text: "Example document", + Model: NEW_MODEL, + }), + }), + Payload: qdrant.NewValueMap(map[string]any{"text": "Example document"}), + }, + }, +}) + +var reEmbedLastOffset *qdrant.PointId +reEmbedBatchSize := uint32(100) +reEmbedReachedEnd := false + +for !reEmbedReachedEnd { + reEmbedScrollResult, err := client.Scroll(context.Background(), &qdrant.ScrollPoints{ + CollectionName: COLLECTION, + Limit: qdrant.PtrOf(reEmbedBatchSize), + Offset: reEmbedLastOffset, + WithPayload: qdrant.NewWithPayload(true), + WithVectors: qdrant.NewWithVectors(false), + }) + + reEmbedRecords := reEmbedScrollResult + + pointVectors := make([]*qdrant.PointVectors, len(reEmbedRecords)) + for idx, record := range reEmbedRecords { + text := "" + if val, ok := record.Payload["text"]; ok { + text = val.GetStringValue() + } + + // Update only the new vector on each point; the old vector and payload are untouched + pointVectors[idx] = &qdrant.PointVectors{ + Id: record.Id, + Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{ + NEW_VECTOR: qdrant.NewVectorDocument(&qdrant.Document{ + Text: text, + Model: NEW_MODEL, + }), + }), + } + } + + client.UpdateVectors(context.Background(), &qdrant.UpdatePointVectors{ + CollectionName: COLLECTION, + Points: pointVectors, + }) + + reEmbedReachedEnd = (reEmbedLastOffset == nil) +} + +oldVectorResults, err := client.Query(context.Background(), &qdrant.QueryPoints{ + CollectionName: COLLECTION, + Query: qdrant.NewQueryDocument(&qdrant.Document{ + Text: "my query", + Model: OLD_MODEL, + }), + Using: qdrant.PtrOf(OLD_VECTOR), + Limit: qdrant.PtrOf(uint64(10)), +}) + +newVectorResults, err := client.Query(context.Background(), &qdrant.QueryPoints{ + CollectionName: COLLECTION, + Query: qdrant.NewQueryDocument(&qdrant.Document{ + Text: "my query", + Model: NEW_MODEL, + }), + Using: qdrant.PtrOf(NEW_VECTOR), + Limit: qdrant.PtrOf(uint64(10)), +}) + +client.DeleteVectorName(context.Background(), &qdrant.DeleteVectorNameRequest{ + CollectionName: COLLECTION, + VectorName: OLD_VECTOR, +}) ``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/java.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/java.md index eed7b0173..c092e7f7f 100644 --- a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/java.md +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/java.md @@ -3,6 +3,7 @@ import static io.qdrant.client.PointIdFactory.id; import static io.qdrant.client.QueryFactory.nearest; import static io.qdrant.client.ValueFactory.value; import static io.qdrant.client.VectorFactory.vector; +import static io.qdrant.client.VectorsFactory.namedVectors; import static io.qdrant.client.VectorsFactory.vectors; import io.qdrant.client.WithPayloadSelectorFactory; import io.qdrant.client.WithVectorsSelectorFactory; @@ -15,6 +16,10 @@ import io.qdrant.client.grpc.JsonWithInt.Value; import io.qdrant.client.grpc.Points.Document; import io.qdrant.client.grpc.Points.PointStruct; import io.qdrant.client.grpc.Points.QueryPoints; +import io.qdrant.client.grpc.Points.CreateVectorNameRequest; +import io.qdrant.client.grpc.Points.DeleteVectorNameRequest; +import io.qdrant.client.grpc.Points.DenseVectorCreationConfig; +import io.qdrant.client.grpc.Points.PointVectors; import io.qdrant.client.grpc.Points.UpsertPoints; import io.qdrant.client.grpc.Points.ScrollPoints; import io.qdrant.client.grpc.Points.UpdateMode; @@ -141,4 +146,113 @@ QueryPoints newRequest = .build(); results = client.queryAsync(newRequest).get(); + +client + .createVectorNameAsync( + CreateVectorNameRequest.newBuilder() + .setCollectionName(COLLECTION) + .setVectorName(NEW_VECTOR) + .setDenseConfig( + DenseVectorCreationConfig.newBuilder() + .setSize(512) // Size of the new embedding vectors + .setDistance(Distance.Cosine) // Similarity function for the new model + .build()) + .build()) + .get(); + +client.upsertAsync(COLLECTION, List.of( + PointStruct.newBuilder() + .setId(id(1)) + .setVectors( + namedVectors( + Map.of( + OLD_VECTOR, vector( + Document.newBuilder() + .setText("Example document") + .setModel(OLD_MODEL) + .build()), + NEW_VECTOR, vector( + Document.newBuilder() + .setText("Example document") + .setModel(NEW_MODEL) + .build())))) + .putAllPayload(Map.of("text", value("Example document"))) + .build())).get(); + +int reEmbedBatchSize = 100; +boolean reEmbedReachedEnd = false; + +var reEmbedScrollBuilder = ScrollPoints.newBuilder() + .setCollectionName(COLLECTION) + .setLimit(reEmbedBatchSize) + .setWithPayload(WithPayloadSelectorFactory.enable(true)) + .setWithVectors(WithVectorsSelectorFactory.enable(false)); + +while (!reEmbedReachedEnd) { + var reEmbedScrollResult = client.scrollAsync(reEmbedScrollBuilder.build()).get(); + var reEmbedRecords = reEmbedScrollResult.getResultList(); + + List pointVectors = new ArrayList<>(); + for (var record : reEmbedRecords) { + String text = record.getPayloadMap().containsKey("text") + ? record.getPayloadMap().get("text").getStringValue() + : ""; + + // Update only the new vector on each point; the old vector and payload are untouched + pointVectors.add( + PointVectors.newBuilder() + .setId(record.getId()) + .setVectors( + namedVectors( + Map.of( + NEW_VECTOR, vector( + Document.newBuilder() + .setText(text) + .setModel(NEW_MODEL) + .build())))) + .build()); + } + + client.updateVectorsAsync(COLLECTION, pointVectors).get(); + + if (reEmbedScrollResult.hasNextPageOffset()) { + reEmbedScrollBuilder.setOffset(reEmbedScrollResult.getNextPageOffset()); + } else { + reEmbedReachedEnd = true; + } +} + +var oldVectorResults = client.queryAsync( + QueryPoints.newBuilder() + .setCollectionName(COLLECTION) + .setQuery( + nearest( + Document.newBuilder() + .setText("my query") + .setModel(OLD_MODEL) + .build())) + .setUsing(OLD_VECTOR) + .setLimit(10) + .build()).get(); + +var newVectorResults = client.queryAsync( + QueryPoints.newBuilder() + .setCollectionName(COLLECTION) + .setQuery( + nearest( + Document.newBuilder() + .setText("my query") + .setModel(NEW_MODEL) + .build())) + .setUsing(NEW_VECTOR) + .setLimit(10) + .build()).get(); + +client + .deleteVectorNameAsync( + DeleteVectorNameRequest.newBuilder() + .setCollectionName(COLLECTION) + .setVectorName(OLD_VECTOR) + .build()) + .get(); ``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/python.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/python.md index 4542e8704..7c989741d 100644 --- a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/python.md +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/python.md @@ -95,4 +95,86 @@ results = client.query_points( query=models.Document(text="my query", model=NEW_MODEL), limit=10, ) + +client.create_vector_name( + collection_name=COLLECTION, + vector_name=NEW_VECTOR, + vector_name_config=models.DenseVectorNameConfig( + dense=models.DenseVectorConfig( + size=512, # Size of the new embedding vectors + distance=models.Distance.COSINE # Similarity function for the new model + ) + ), +) + +client.upsert( + collection_name=COLLECTION, + points=[ + models.PointStruct( + id=1, + vector={ + OLD_VECTOR: models.Document( + text="Example document", + model=OLD_MODEL, + ), + NEW_VECTOR: models.Document( + text="Example document", + model=NEW_MODEL, + ), + }, + payload={"text": "Example document"} + ) + ] +) + +last_offset = None +batch_size = 100 +reached_end = False + +while not reached_end: + records, last_offset = client.scroll( + collection_name=COLLECTION, + limit=batch_size, + offset=last_offset, + with_payload=True, + with_vectors=False, + ) + + # Update only the new vector on each point; the old vector and payload are untouched + client.update_vectors( + collection_name=COLLECTION, + points=[ + models.PointVectors( + id=record.id, + vector={ + NEW_VECTOR: models.Document( + text=(record.payload or {}).get("text", ""), + model=NEW_MODEL, + ) + }, + ) + for record in records + ], + ) + + reached_end = last_offset is None + +results = client.query_points( + collection_name=COLLECTION, + query=models.Document(text="my query", model=OLD_MODEL), + using=OLD_VECTOR, + limit=10, +) + +results = client.query_points( + collection_name=COLLECTION, + query=models.Document(text="my query", model=NEW_MODEL), + using=NEW_VECTOR, + limit=10, +) + +client.delete_vector_name( + collection_name=COLLECTION, + vector_name=OLD_VECTOR, +) ``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/re-embed-existing/csharp.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/re-embed-existing/csharp.md new file mode 100644 index 000000000..7dea57d86 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/re-embed-existing/csharp.md @@ -0,0 +1,41 @@ +```csharp +PointId? reEmbedLastOffset = null; +uint reEmbedBatchSize = 100; +bool reEmbedReachedEnd = false; + +while (!reEmbedReachedEnd) +{ + var reEmbedScrollResult = await client.ScrollAsync( + collectionName: COLLECTION, + limit: reEmbedBatchSize, + offset: reEmbedLastOffset, + payloadSelector: true, + vectorsSelector: false + ); + + var reEmbedRecords = reEmbedScrollResult.Result; + reEmbedLastOffset = reEmbedScrollResult.NextPageOffset; + + var pointVectors = new List(); + foreach (var record in reEmbedRecords) + { + var text = record.Payload.ContainsKey("text") + ? record.Payload["text"].StringValue + : ""; + + // Update only the new vector on each point; the old vector and payload are untouched + pointVectors.Add(new PointVectors + { + Id = record.Id, + Vectors = new Dictionary + { + [NEW_VECTOR] = new Document { Text = text, Model = NEW_MODEL } + } + }); + } + + await client.UpdateVectorsAsync(collectionName: COLLECTION, points: pointVectors); + + reEmbedReachedEnd = (reEmbedLastOffset == null); +} +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/re-embed-existing/go.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/re-embed-existing/go.md new file mode 100644 index 000000000..7d999df4d --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/re-embed-existing/go.md @@ -0,0 +1,43 @@ +```go +var reEmbedLastOffset *qdrant.PointId +reEmbedBatchSize := uint32(100) +reEmbedReachedEnd := false + +for !reEmbedReachedEnd { + reEmbedScrollResult, err := client.Scroll(context.Background(), &qdrant.ScrollPoints{ + CollectionName: COLLECTION, + Limit: qdrant.PtrOf(reEmbedBatchSize), + Offset: reEmbedLastOffset, + WithPayload: qdrant.NewWithPayload(true), + WithVectors: qdrant.NewWithVectors(false), + }) + + reEmbedRecords := reEmbedScrollResult + + pointVectors := make([]*qdrant.PointVectors, len(reEmbedRecords)) + for idx, record := range reEmbedRecords { + text := "" + if val, ok := record.Payload["text"]; ok { + text = val.GetStringValue() + } + + // Update only the new vector on each point; the old vector and payload are untouched + pointVectors[idx] = &qdrant.PointVectors{ + Id: record.Id, + Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{ + NEW_VECTOR: qdrant.NewVectorDocument(&qdrant.Document{ + Text: text, + Model: NEW_MODEL, + }), + }), + } + } + + client.UpdateVectors(context.Background(), &qdrant.UpdatePointVectors{ + CollectionName: COLLECTION, + Points: pointVectors, + }) + + reEmbedReachedEnd = (reEmbedLastOffset == nil) +} +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/re-embed-existing/java.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/re-embed-existing/java.md new file mode 100644 index 000000000..2e89ccccd --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/re-embed-existing/java.md @@ -0,0 +1,44 @@ +```java +int reEmbedBatchSize = 100; +boolean reEmbedReachedEnd = false; + +var reEmbedScrollBuilder = ScrollPoints.newBuilder() + .setCollectionName(COLLECTION) + .setLimit(reEmbedBatchSize) + .setWithPayload(WithPayloadSelectorFactory.enable(true)) + .setWithVectors(WithVectorsSelectorFactory.enable(false)); + +while (!reEmbedReachedEnd) { + var reEmbedScrollResult = client.scrollAsync(reEmbedScrollBuilder.build()).get(); + var reEmbedRecords = reEmbedScrollResult.getResultList(); + + List pointVectors = new ArrayList<>(); + for (var record : reEmbedRecords) { + String text = record.getPayloadMap().containsKey("text") + ? record.getPayloadMap().get("text").getStringValue() + : ""; + + // Update only the new vector on each point; the old vector and payload are untouched + pointVectors.add( + PointVectors.newBuilder() + .setId(record.getId()) + .setVectors( + namedVectors( + Map.of( + NEW_VECTOR, vector( + Document.newBuilder() + .setText(text) + .setModel(NEW_MODEL) + .build())))) + .build()); + } + + client.updateVectorsAsync(COLLECTION, pointVectors).get(); + + if (reEmbedScrollResult.hasNextPageOffset()) { + reEmbedScrollBuilder.setOffset(reEmbedScrollResult.getNextPageOffset()); + } else { + reEmbedReachedEnd = true; + } +} +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/re-embed-existing/python.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/re-embed-existing/python.md new file mode 100644 index 000000000..ab0ba13b8 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/re-embed-existing/python.md @@ -0,0 +1,33 @@ +```python +last_offset = None +batch_size = 100 +reached_end = False + +while not reached_end: + records, last_offset = client.scroll( + collection_name=COLLECTION, + limit=batch_size, + offset=last_offset, + with_payload=True, + with_vectors=False, + ) + + # Update only the new vector on each point; the old vector and payload are untouched + client.update_vectors( + collection_name=COLLECTION, + points=[ + models.PointVectors( + id=record.id, + vector={ + NEW_VECTOR: models.Document( + text=(record.payload or {}).get("text", ""), + model=NEW_MODEL, + ) + }, + ) + for record in records + ], + ) + + reached_end = last_offset is None +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/re-embed-existing/rust.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/re-embed-existing/rust.md new file mode 100644 index 000000000..4edef84f7 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/re-embed-existing/rust.md @@ -0,0 +1,47 @@ +```rust +let mut last_offset = None; +let batch_size = 100; + +loop { + let mut scroll_builder = ScrollPointsBuilder::new(collection) + .limit(batch_size) + .with_payload(true) + .with_vectors(false); + + if let Some(offset) = last_offset { + scroll_builder = scroll_builder.offset(offset); + } + + let scroll_result = client.scroll(scroll_builder).await?; + let records = scroll_result.result; + last_offset = scroll_result.next_page_offset; + + // Update only the new vector on each point; the old vector and payload are untouched + let point_vectors: Vec = records + .iter() + .map(|record| PointVectors { + id: record.id.clone(), + vectors: Some( + HashMap::::from([( + new_vector.to_string(), + Document::new( + record.payload.get("text") + .and_then(|v| v.as_str()) + .map_or("", |v| v), + new_model, + ), + )]) + .into(), + ), + }) + .collect(); + + client + .update_vectors(UpdatePointVectorsBuilder::new(collection, point_vectors)) + .await?; + + if last_offset.is_none() { + break; + } +} +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/re-embed-existing/typescript.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/re-embed-existing/typescript.md new file mode 100644 index 000000000..ad04a7008 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/re-embed-existing/typescript.md @@ -0,0 +1,32 @@ +```typescript +let reEmbedLastOffset: number | string | undefined = undefined; +const reEmbedBatchSize = 100; +let reEmbedReachedEnd = false; + +while (!reEmbedReachedEnd) { + const reEmbedScrollResult = await client.scroll(COLLECTION, { + limit: reEmbedBatchSize, + offset: reEmbedLastOffset, + with_payload: true, + with_vector: false, + }); + + const records = reEmbedScrollResult.points; + reEmbedLastOffset = reEmbedScrollResult.next_page_offset as number | string | undefined; + + // Update only the new vector on each point; the old vector and payload are untouched + await client.updateVectors(COLLECTION, { + points: records.map((record) => ({ + id: record.id, + vector: { + [NEW_VECTOR]: { + text: ((record.payload?.text as string) ?? ""), + model: NEW_MODEL, + }, + }, + })), + }); + + reEmbedReachedEnd = reEmbedLastOffset == null; +} +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/rust.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/rust.md index fdcaffb41..3469b54a9 100644 --- a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/rust.md +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/rust.md @@ -1,7 +1,11 @@ ```rust +use std::collections::HashMap; + use qdrant_client::qdrant::{ - CreateCollectionBuilder, Distance, Document, PointStruct, Query, QueryPointsBuilder, - ScrollPointsBuilder, UpdateMode, UpsertPointsBuilder, VectorParamsBuilder, + CreateCollectionBuilder, CreateVectorNameRequestBuilder, DeleteVectorNameRequestBuilder, + DenseVectorCreationConfigBuilder, Distance, Document, NamedVectors, PointStruct, PointVectors, + Query, QueryPointsBuilder, ScrollPointsBuilder, UpdateMode, UpdatePointVectorsBuilder, + UpsertPointsBuilder, VectorParamsBuilder, }; use qdrant_client::Qdrant; @@ -107,4 +111,112 @@ let results = client .limit(10), ) .await?; + +client + .create_vector_name( + CreateVectorNameRequestBuilder::new( + collection, + new_vector, + DenseVectorCreationConfigBuilder::new(512, Distance::Cosine), // Size of the new embedding vectors + ), + ) + .await?; + +client + .upsert_points(UpsertPointsBuilder::new( + collection, + vec![PointStruct::new( + 1, + NamedVectors::default() + .add_vector( + old_vector, + Document { + text: "Example document".into(), + model: old_model.into(), + ..Default::default() + }, + ) + .add_vector( + new_vector, + Document { + text: "Example document".into(), + model: new_model.into(), + ..Default::default() + }, + ), + [("text", "Example document".into())], + )], + )) + .await?; + +let mut last_offset = None; +let batch_size = 100; + +loop { + let mut scroll_builder = ScrollPointsBuilder::new(collection) + .limit(batch_size) + .with_payload(true) + .with_vectors(false); + + if let Some(offset) = last_offset { + scroll_builder = scroll_builder.offset(offset); + } + + let scroll_result = client.scroll(scroll_builder).await?; + let records = scroll_result.result; + last_offset = scroll_result.next_page_offset; + + // Update only the new vector on each point; the old vector and payload are untouched + let point_vectors: Vec = records + .iter() + .map(|record| PointVectors { + id: record.id.clone(), + vectors: Some( + HashMap::::from([( + new_vector.to_string(), + Document::new( + record.payload.get("text") + .and_then(|v| v.as_str()) + .map_or("", |v| v), + new_model, + ), + )]) + .into(), + ), + }) + .collect(); + + client + .update_vectors(UpdatePointVectorsBuilder::new(collection, point_vectors)) + .await?; + + if last_offset.is_none() { + break; + } +} + +let old_vector_results = client + .query( + QueryPointsBuilder::new(collection) + .query(Query::new_nearest(Document::new("my query", old_model))) + .using(old_vector) + .limit(10), + ) + .await?; + +let new_vector_results = client + .query( + QueryPointsBuilder::new(collection) + .query(Query::new_nearest(Document::new("my query", new_model))) + .using(new_vector) + .limit(10), + ) + .await?; + +client + .delete_vector_name(DeleteVectorNameRequestBuilder::new( + collection, + old_vector, + )) + .await?; ``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-new-vector/csharp.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-new-vector/csharp.md new file mode 100644 index 000000000..52038bc90 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-new-vector/csharp.md @@ -0,0 +1,8 @@ +```csharp +var newVectorResults = await client.QueryAsync( + collectionName: COLLECTION, + query: new Document { Text = "my query", Model = NEW_MODEL }, + usingVector: NEW_VECTOR, + limit: 10 +); +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-new-vector/go.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-new-vector/go.md new file mode 100644 index 000000000..eb55ffd1c --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-new-vector/go.md @@ -0,0 +1,11 @@ +```go +newVectorResults, err := client.Query(context.Background(), &qdrant.QueryPoints{ + CollectionName: COLLECTION, + Query: qdrant.NewQueryDocument(&qdrant.Document{ + Text: "my query", + Model: NEW_MODEL, + }), + Using: qdrant.PtrOf(NEW_VECTOR), + Limit: qdrant.PtrOf(uint64(10)), +}) +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-new-vector/java.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-new-vector/java.md new file mode 100644 index 000000000..17089de49 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-new-vector/java.md @@ -0,0 +1,14 @@ +```java +var newVectorResults = client.queryAsync( + QueryPoints.newBuilder() + .setCollectionName(COLLECTION) + .setQuery( + nearest( + Document.newBuilder() + .setText("my query") + .setModel(NEW_MODEL) + .build())) + .setUsing(NEW_VECTOR) + .setLimit(10) + .build()).get(); +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-new-vector/python.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-new-vector/python.md new file mode 100644 index 000000000..d3b91fd99 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-new-vector/python.md @@ -0,0 +1,8 @@ +```python +results = client.query_points( + collection_name=COLLECTION, + query=models.Document(text="my query", model=NEW_MODEL), + using=NEW_VECTOR, + limit=10, +) +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-new-vector/rust.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-new-vector/rust.md new file mode 100644 index 000000000..ed5c9f444 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-new-vector/rust.md @@ -0,0 +1,10 @@ +```rust +let new_vector_results = client + .query( + QueryPointsBuilder::new(collection) + .query(Query::new_nearest(Document::new("my query", new_model))) + .using(new_vector) + .limit(10), + ) + .await?; +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-new-vector/typescript.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-new-vector/typescript.md new file mode 100644 index 000000000..1c1768780 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-new-vector/typescript.md @@ -0,0 +1,10 @@ +```typescript +const newVectorResults = await client.query(COLLECTION, { + query: { + text: "my query", + model: NEW_MODEL, + }, + using: NEW_VECTOR, + limit: 10, +}); +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-old-vector/csharp.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-old-vector/csharp.md new file mode 100644 index 000000000..51a9a5996 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-old-vector/csharp.md @@ -0,0 +1,8 @@ +```csharp +var oldVectorResults = await client.QueryAsync( + collectionName: COLLECTION, + query: new Document { Text = "my query", Model = OLD_MODEL }, + usingVector: OLD_VECTOR, + limit: 10 +); +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-old-vector/go.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-old-vector/go.md new file mode 100644 index 000000000..c90a15749 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-old-vector/go.md @@ -0,0 +1,11 @@ +```go +oldVectorResults, err := client.Query(context.Background(), &qdrant.QueryPoints{ + CollectionName: COLLECTION, + Query: qdrant.NewQueryDocument(&qdrant.Document{ + Text: "my query", + Model: OLD_MODEL, + }), + Using: qdrant.PtrOf(OLD_VECTOR), + Limit: qdrant.PtrOf(uint64(10)), +}) +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-old-vector/java.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-old-vector/java.md new file mode 100644 index 000000000..8bfb6416d --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-old-vector/java.md @@ -0,0 +1,14 @@ +```java +var oldVectorResults = client.queryAsync( + QueryPoints.newBuilder() + .setCollectionName(COLLECTION) + .setQuery( + nearest( + Document.newBuilder() + .setText("my query") + .setModel(OLD_MODEL) + .build())) + .setUsing(OLD_VECTOR) + .setLimit(10) + .build()).get(); +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-old-vector/python.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-old-vector/python.md new file mode 100644 index 000000000..f3eb94422 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-old-vector/python.md @@ -0,0 +1,8 @@ +```python +results = client.query_points( + collection_name=COLLECTION, + query=models.Document(text="my query", model=OLD_MODEL), + using=OLD_VECTOR, + limit=10, +) +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-old-vector/rust.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-old-vector/rust.md new file mode 100644 index 000000000..31452d7f4 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-old-vector/rust.md @@ -0,0 +1,10 @@ +```rust +let old_vector_results = client + .query( + QueryPointsBuilder::new(collection) + .query(Query::new_nearest(Document::new("my query", old_model))) + .using(old_vector) + .limit(10), + ) + .await?; +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-old-vector/typescript.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-old-vector/typescript.md new file mode 100644 index 000000000..4736a98e5 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/search-with-old-vector/typescript.md @@ -0,0 +1,10 @@ +```typescript +const oldVectorResults = await client.query(COLLECTION, { + query: { + text: "my query", + model: OLD_MODEL, + }, + using: OLD_VECTOR, + limit: 10, +}); +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/typescript.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/typescript.md index a7e3347be..12799edfd 100644 --- a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/typescript.md +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/typescript.md @@ -93,4 +93,81 @@ const resultsNew = await client.query(NEW_COLLECTION, { }, limit: 10, }); + +await client.createVectorName(COLLECTION, NEW_VECTOR, { + dense: { + size: 512, // Size of the new embedding vectors + distance: "Cosine", // Similarity function for the new model + }, +}); + +await client.upsert(COLLECTION, { + points: [ + { + id: 1, + vector: { + [OLD_VECTOR]: { + text: "Example document", + model: OLD_MODEL, + }, + [NEW_VECTOR]: { + text: "Example document", + model: NEW_MODEL, + }, + }, + payload: { text: "Example document" }, + }, + ], +}); + +let reEmbedLastOffset: number | string | undefined = undefined; +const reEmbedBatchSize = 100; +let reEmbedReachedEnd = false; + +while (!reEmbedReachedEnd) { + const reEmbedScrollResult = await client.scroll(COLLECTION, { + limit: reEmbedBatchSize, + offset: reEmbedLastOffset, + with_payload: true, + with_vector: false, + }); + + const records = reEmbedScrollResult.points; + reEmbedLastOffset = reEmbedScrollResult.next_page_offset as number | string | undefined; + + // Update only the new vector on each point; the old vector and payload are untouched + await client.updateVectors(COLLECTION, { + points: records.map((record) => ({ + id: record.id, + vector: { + [NEW_VECTOR]: { + text: ((record.payload?.text as string) ?? ""), + model: NEW_MODEL, + }, + }, + })), + }); + + reEmbedReachedEnd = reEmbedLastOffset == null; +} + +const oldVectorResults = await client.query(COLLECTION, { + query: { + text: "my query", + model: OLD_MODEL, + }, + using: OLD_VECTOR, + limit: 10, +}); + +const newVectorResults = await client.query(COLLECTION, { + query: { + text: "my query", + model: NEW_MODEL, + }, + using: NEW_VECTOR, + limit: 10, +}); + +await client.deleteVectorName(COLLECTION, OLD_VECTOR); ``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/upsert-both-vectors/csharp.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/upsert-both-vectors/csharp.md new file mode 100644 index 000000000..d54af3c55 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/upsert-both-vectors/csharp.md @@ -0,0 +1,18 @@ +```csharp +await client.UpsertAsync( + collectionName: COLLECTION, + points: new List + { + new() + { + Id = 1, + Vectors = new Dictionary + { + [OLD_VECTOR] = new Document { Text = "Example document", Model = OLD_MODEL }, + [NEW_VECTOR] = new Document { Text = "Example document", Model = NEW_MODEL }, + }, + Payload = { ["text"] = "Example document" } + } + } +); +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/upsert-both-vectors/go.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/upsert-both-vectors/go.md new file mode 100644 index 000000000..49bca8753 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/upsert-both-vectors/go.md @@ -0,0 +1,21 @@ +```go +client.Upsert(context.Background(), &qdrant.UpsertPoints{ + CollectionName: COLLECTION, + Points: []*qdrant.PointStruct{ + { + Id: qdrant.NewIDNum(1), + Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{ + OLD_VECTOR: qdrant.NewVectorDocument(&qdrant.Document{ + Text: "Example document", + Model: OLD_MODEL, + }), + NEW_VECTOR: qdrant.NewVectorDocument(&qdrant.Document{ + Text: "Example document", + Model: NEW_MODEL, + }), + }), + Payload: qdrant.NewValueMap(map[string]any{"text": "Example document"}), + }, + }, +}) +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/upsert-both-vectors/java.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/upsert-both-vectors/java.md new file mode 100644 index 000000000..92198e436 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/upsert-both-vectors/java.md @@ -0,0 +1,20 @@ +```java +client.upsertAsync(COLLECTION, List.of( + PointStruct.newBuilder() + .setId(id(1)) + .setVectors( + namedVectors( + Map.of( + OLD_VECTOR, vector( + Document.newBuilder() + .setText("Example document") + .setModel(OLD_MODEL) + .build()), + NEW_VECTOR, vector( + Document.newBuilder() + .setText("Example document") + .setModel(NEW_MODEL) + .build())))) + .putAllPayload(Map.of("text", value("Example document"))) + .build())).get(); +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/upsert-both-vectors/python.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/upsert-both-vectors/python.md new file mode 100644 index 000000000..d52efca4f --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/upsert-both-vectors/python.md @@ -0,0 +1,21 @@ +```python +client.upsert( + collection_name=COLLECTION, + points=[ + models.PointStruct( + id=1, + vector={ + OLD_VECTOR: models.Document( + text="Example document", + model=OLD_MODEL, + ), + NEW_VECTOR: models.Document( + text="Example document", + model=NEW_MODEL, + ), + }, + payload={"text": "Example document"} + ) + ] +) +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/upsert-both-vectors/rust.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/upsert-both-vectors/rust.md new file mode 100644 index 000000000..385f8bfb7 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/upsert-both-vectors/rust.md @@ -0,0 +1,28 @@ +```rust +client + .upsert_points(UpsertPointsBuilder::new( + collection, + vec![PointStruct::new( + 1, + NamedVectors::default() + .add_vector( + old_vector, + Document { + text: "Example document".into(), + model: old_model.into(), + ..Default::default() + }, + ) + .add_vector( + new_vector, + Document { + text: "Example document".into(), + model: new_model.into(), + ..Default::default() + }, + ), + [("text", "Example document".into())], + )], + )) + .await?; +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/upsert-both-vectors/typescript.md b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/upsert-both-vectors/typescript.md new file mode 100644 index 000000000..841dd58f0 --- /dev/null +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/generated/upsert-both-vectors/typescript.md @@ -0,0 +1,20 @@ +```typescript +await client.upsert(COLLECTION, { + points: [ + { + id: 1, + vector: { + [OLD_VECTOR]: { + text: "Example document", + model: OLD_MODEL, + }, + [NEW_VECTOR]: { + text: "Example document", + model: NEW_MODEL, + }, + }, + payload: { text: "Example document" }, + }, + ], +}); +``` diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/go.go b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/go.go index 1500e99ac..39a36a2c0 100644 --- a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/go.go +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/go.go @@ -23,6 +23,10 @@ func Main() { OLD_MODEL := "sentence-transformers/all-minilm-l6-v2" NEW_MODEL := "qdrant/clip-vit-b-32-text" + + COLLECTION := "my_collection" + OLD_VECTOR := "old-model" + NEW_VECTOR := "new-model" // @hide-end // @block-start create-new-collection @@ -164,4 +168,135 @@ func Main() { } _ = results // @hide-end + + // @block-start add-named-vector + client.CreateVectorName(context.Background(), &qdrant.CreateVectorNameRequest{ + CollectionName: COLLECTION, + VectorName: NEW_VECTOR, + VectorConfig: &qdrant.CreateVectorNameRequest_DenseConfig{ + DenseConfig: &qdrant.DenseVectorCreationConfig{ + Size: 512, // Size of the new embedding vectors + Distance: qdrant.Distance_Cosine, + }, + }, + }) + // @block-end add-named-vector + + // @block-start upsert-both-vectors + client.Upsert(context.Background(), &qdrant.UpsertPoints{ + CollectionName: COLLECTION, + Points: []*qdrant.PointStruct{ + { + Id: qdrant.NewIDNum(1), + Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{ + OLD_VECTOR: qdrant.NewVectorDocument(&qdrant.Document{ + Text: "Example document", + Model: OLD_MODEL, + }), + NEW_VECTOR: qdrant.NewVectorDocument(&qdrant.Document{ + Text: "Example document", + Model: NEW_MODEL, + }), + }), + Payload: qdrant.NewValueMap(map[string]any{"text": "Example document"}), + }, + }, + }) + // @block-end upsert-both-vectors + + // @block-start re-embed-existing + var reEmbedLastOffset *qdrant.PointId + reEmbedBatchSize := uint32(100) + reEmbedReachedEnd := false + + for !reEmbedReachedEnd { + reEmbedScrollResult, err := client.Scroll(context.Background(), &qdrant.ScrollPoints{ + CollectionName: COLLECTION, + Limit: qdrant.PtrOf(reEmbedBatchSize), + Offset: reEmbedLastOffset, + WithPayload: qdrant.NewWithPayload(true), + WithVectors: qdrant.NewWithVectors(false), + }) + + // @hide-start + if err != nil { + panic(err) + } + // @hide-end + + reEmbedRecords := reEmbedScrollResult + reEmbedLastOffset = reEmbedScrollResult[len(reEmbedScrollResult)-1].Id // @hide + + pointVectors := make([]*qdrant.PointVectors, len(reEmbedRecords)) + for idx, record := range reEmbedRecords { + text := "" + if val, ok := record.Payload["text"]; ok { + text = val.GetStringValue() + } + + // Update only the new vector on each point; the old vector and payload are untouched + pointVectors[idx] = &qdrant.PointVectors{ + Id: record.Id, + Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{ + NEW_VECTOR: qdrant.NewVectorDocument(&qdrant.Document{ + Text: text, + Model: NEW_MODEL, + }), + }), + } + } + + client.UpdateVectors(context.Background(), &qdrant.UpdatePointVectors{ + CollectionName: COLLECTION, + Points: pointVectors, + }) + + reEmbedReachedEnd = (reEmbedLastOffset == nil) + } + // @block-end re-embed-existing + + // @block-start search-with-old-vector + oldVectorResults, err := client.Query(context.Background(), &qdrant.QueryPoints{ + CollectionName: COLLECTION, + Query: qdrant.NewQueryDocument(&qdrant.Document{ + Text: "my query", + Model: OLD_MODEL, + }), + Using: qdrant.PtrOf(OLD_VECTOR), + Limit: qdrant.PtrOf(uint64(10)), + }) + // @block-end search-with-old-vector + + // @hide-start + if err != nil { + panic(err) + } + _ = oldVectorResults + // @hide-end + + // @block-start search-with-new-vector + newVectorResults, err := client.Query(context.Background(), &qdrant.QueryPoints{ + CollectionName: COLLECTION, + Query: qdrant.NewQueryDocument(&qdrant.Document{ + Text: "my query", + Model: NEW_MODEL, + }), + Using: qdrant.PtrOf(NEW_VECTOR), + Limit: qdrant.PtrOf(uint64(10)), + }) + // @block-end search-with-new-vector + + // @hide-start + if err != nil { + panic(err) + } + _ = newVectorResults + // @hide-end + + // @block-start delete-old-named-vector + client.DeleteVectorName(context.Background(), &qdrant.DeleteVectorNameRequest{ + CollectionName: COLLECTION, + VectorName: OLD_VECTOR, + }) + // @block-end delete-old-named-vector } diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/java.java b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/java.java index 8002b7ae8..74773dc42 100644 --- a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/java.java +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/java.java @@ -4,6 +4,7 @@ import static io.qdrant.client.PointIdFactory.id; import static io.qdrant.client.QueryFactory.nearest; import static io.qdrant.client.ValueFactory.value; import static io.qdrant.client.VectorFactory.vector; +import static io.qdrant.client.VectorsFactory.namedVectors; import static io.qdrant.client.VectorsFactory.vectors; import io.qdrant.client.WithPayloadSelectorFactory; import io.qdrant.client.WithVectorsSelectorFactory; @@ -16,6 +17,10 @@ import io.qdrant.client.grpc.JsonWithInt.Value; import io.qdrant.client.grpc.Points.Document; import io.qdrant.client.grpc.Points.PointStruct; import io.qdrant.client.grpc.Points.QueryPoints; +import io.qdrant.client.grpc.Points.CreateVectorNameRequest; +import io.qdrant.client.grpc.Points.DeleteVectorNameRequest; +import io.qdrant.client.grpc.Points.DenseVectorCreationConfig; +import io.qdrant.client.grpc.Points.PointVectors; import io.qdrant.client.grpc.Points.UpsertPoints; import io.qdrant.client.grpc.Points.ScrollPoints; import io.qdrant.client.grpc.Points.UpdateMode; @@ -41,6 +46,10 @@ public class Snippet { String OLD_MODEL = "sentence-transformers/all-minilm-l6-v2"; String NEW_MODEL = "qdrant/clip-vit-b-32-text"; + + String COLLECTION = "my_collection"; + String OLD_VECTOR = "old-model"; + String NEW_VECTOR = "new-model"; // @hide-end // @block-start create-new-collection @@ -174,6 +183,127 @@ public class Snippet { results = client.queryAsync(newRequest).get(); // @block-end search-new-collection + + // @block-start add-named-vector + client + .createVectorNameAsync( + CreateVectorNameRequest.newBuilder() + .setCollectionName(COLLECTION) + .setVectorName(NEW_VECTOR) + .setDenseConfig( + DenseVectorCreationConfig.newBuilder() + .setSize(512) // Size of the new embedding vectors + .setDistance(Distance.Cosine) // Similarity function for the new model + .build()) + .build()) + .get(); + // @block-end add-named-vector + + // @block-start upsert-both-vectors + client.upsertAsync(COLLECTION, List.of( + PointStruct.newBuilder() + .setId(id(1)) + .setVectors( + namedVectors( + Map.of( + OLD_VECTOR, vector( + Document.newBuilder() + .setText("Example document") + .setModel(OLD_MODEL) + .build()), + NEW_VECTOR, vector( + Document.newBuilder() + .setText("Example document") + .setModel(NEW_MODEL) + .build())))) + .putAllPayload(Map.of("text", value("Example document"))) + .build())).get(); + // @block-end upsert-both-vectors + + // @block-start re-embed-existing + int reEmbedBatchSize = 100; + boolean reEmbedReachedEnd = false; + + var reEmbedScrollBuilder = ScrollPoints.newBuilder() + .setCollectionName(COLLECTION) + .setLimit(reEmbedBatchSize) + .setWithPayload(WithPayloadSelectorFactory.enable(true)) + .setWithVectors(WithVectorsSelectorFactory.enable(false)); + + while (!reEmbedReachedEnd) { + var reEmbedScrollResult = client.scrollAsync(reEmbedScrollBuilder.build()).get(); + var reEmbedRecords = reEmbedScrollResult.getResultList(); + + List pointVectors = new ArrayList<>(); + for (var record : reEmbedRecords) { + String text = record.getPayloadMap().containsKey("text") + ? record.getPayloadMap().get("text").getStringValue() + : ""; + + // Update only the new vector on each point; the old vector and payload are untouched + pointVectors.add( + PointVectors.newBuilder() + .setId(record.getId()) + .setVectors( + namedVectors( + Map.of( + NEW_VECTOR, vector( + Document.newBuilder() + .setText(text) + .setModel(NEW_MODEL) + .build())))) + .build()); + } + + client.updateVectorsAsync(COLLECTION, pointVectors).get(); + + if (reEmbedScrollResult.hasNextPageOffset()) { + reEmbedScrollBuilder.setOffset(reEmbedScrollResult.getNextPageOffset()); + } else { + reEmbedReachedEnd = true; + } + } + // @block-end re-embed-existing + + // @block-start search-with-old-vector + var oldVectorResults = client.queryAsync( + QueryPoints.newBuilder() + .setCollectionName(COLLECTION) + .setQuery( + nearest( + Document.newBuilder() + .setText("my query") + .setModel(OLD_MODEL) + .build())) + .setUsing(OLD_VECTOR) + .setLimit(10) + .build()).get(); + // @block-end search-with-old-vector + + // @block-start search-with-new-vector + var newVectorResults = client.queryAsync( + QueryPoints.newBuilder() + .setCollectionName(COLLECTION) + .setQuery( + nearest( + Document.newBuilder() + .setText("my query") + .setModel(NEW_MODEL) + .build())) + .setUsing(NEW_VECTOR) + .setLimit(10) + .build()).get(); + // @block-end search-with-new-vector + + // @block-start delete-old-named-vector + client + .deleteVectorNameAsync( + DeleteVectorNameRequest.newBuilder() + .setCollectionName(COLLECTION) + .setVectorName(OLD_VECTOR) + .build()) + .get(); + // @block-end delete-old-named-vector } } diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/python.py b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/python.py index d99dd534d..c5e92520b 100644 --- a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/python.py +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/python.py @@ -11,6 +11,10 @@ OLD_COLLECTION="old_collection" OLD_MODEL="sentence-transformers/all-minilm-l6-v2" NEW_MODEL="qdrant/clip-vit-b-32-text" + +COLLECTION="my_collection" +OLD_VECTOR="old-model" +NEW_VECTOR="new-model" # @hide-end # @block-start create-new-collection @@ -119,3 +123,97 @@ results = client.query_points( limit=10, ) # @block-end search-new-collection + +# @block-start add-named-vector +client.create_vector_name( + collection_name=COLLECTION, + vector_name=NEW_VECTOR, + vector_name_config=models.DenseVectorNameConfig( + dense=models.DenseVectorConfig( + size=512, # Size of the new embedding vectors + distance=models.Distance.COSINE # Similarity function for the new model + ) + ), +) +# @block-end add-named-vector + +# @block-start upsert-both-vectors +client.upsert( + collection_name=COLLECTION, + points=[ + models.PointStruct( + id=1, + vector={ + OLD_VECTOR: models.Document( + text="Example document", + model=OLD_MODEL, + ), + NEW_VECTOR: models.Document( + text="Example document", + model=NEW_MODEL, + ), + }, + payload={"text": "Example document"} + ) + ] +) +# @block-end upsert-both-vectors + +# @block-start re-embed-existing +last_offset = None +batch_size = 100 +reached_end = False + +while not reached_end: + records, last_offset = client.scroll( + collection_name=COLLECTION, + limit=batch_size, + offset=last_offset, + with_payload=True, + with_vectors=False, + ) + + # Update only the new vector on each point; the old vector and payload are untouched + client.update_vectors( + collection_name=COLLECTION, + points=[ + models.PointVectors( + id=record.id, + vector={ + NEW_VECTOR: models.Document( + text=(record.payload or {}).get("text", ""), + model=NEW_MODEL, + ) + }, + ) + for record in records + ], + ) + + reached_end = last_offset is None +# @block-end re-embed-existing + +# @block-start search-with-old-vector +results = client.query_points( + collection_name=COLLECTION, + query=models.Document(text="my query", model=OLD_MODEL), + using=OLD_VECTOR, + limit=10, +) +# @block-end search-with-old-vector + +# @block-start search-with-new-vector +results = client.query_points( + collection_name=COLLECTION, + query=models.Document(text="my query", model=NEW_MODEL), + using=NEW_VECTOR, + limit=10, +) +# @block-end search-with-new-vector + +# @block-start delete-old-named-vector +client.delete_vector_name( + collection_name=COLLECTION, + vector_name=OLD_VECTOR, +) +# @block-end delete-old-named-vector diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/rust.rs b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/rust.rs index 44ef2a330..93df4e562 100644 --- a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/rust.rs +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/rust.rs @@ -1,6 +1,10 @@ +use std::collections::HashMap; + use qdrant_client::qdrant::{ - CreateCollectionBuilder, Distance, Document, PointStruct, Query, QueryPointsBuilder, - ScrollPointsBuilder, UpdateMode, UpsertPointsBuilder, VectorParamsBuilder, + CreateCollectionBuilder, CreateVectorNameRequestBuilder, DeleteVectorNameRequestBuilder, + DenseVectorCreationConfigBuilder, Distance, Document, NamedVectors, PointStruct, PointVectors, + Query, QueryPointsBuilder, ScrollPointsBuilder, UpdateMode, UpdatePointVectorsBuilder, + UpsertPointsBuilder, VectorParamsBuilder, }; use qdrant_client::Qdrant; @@ -18,6 +22,10 @@ pub async fn main() -> anyhow::Result<()> { let old_model = "sentence-transformers/all-minilm-l6-v2"; let new_model = "qdrant/clip-vit-b-32-text"; + + let collection = "my_collection"; + let old_vector = "old-model"; + let new_vector = "new-model"; // @hide-end // @block-start create-new-collection @@ -143,5 +151,133 @@ pub async fn main() -> anyhow::Result<()> { _ = results; // @hide-end + // @block-start add-named-vector + client + .create_vector_name( + CreateVectorNameRequestBuilder::new( + collection, + new_vector, + DenseVectorCreationConfigBuilder::new(512, Distance::Cosine), // Size of the new embedding vectors + ), + ) + .await?; + // @block-end add-named-vector + + // @block-start upsert-both-vectors + client + .upsert_points(UpsertPointsBuilder::new( + collection, + vec![PointStruct::new( + 1, + NamedVectors::default() + .add_vector( + old_vector, + Document { + text: "Example document".into(), + model: old_model.into(), + ..Default::default() + }, + ) + .add_vector( + new_vector, + Document { + text: "Example document".into(), + model: new_model.into(), + ..Default::default() + }, + ), + [("text", "Example document".into())], + )], + )) + .await?; + // @block-end upsert-both-vectors + + // @block-start re-embed-existing + let mut last_offset = None; + let batch_size = 100; + + loop { + let mut scroll_builder = ScrollPointsBuilder::new(collection) + .limit(batch_size) + .with_payload(true) + .with_vectors(false); + + if let Some(offset) = last_offset { + scroll_builder = scroll_builder.offset(offset); + } + + let scroll_result = client.scroll(scroll_builder).await?; + let records = scroll_result.result; + last_offset = scroll_result.next_page_offset; + + // Update only the new vector on each point; the old vector and payload are untouched + let point_vectors: Vec = records + .iter() + .map(|record| PointVectors { + id: record.id.clone(), + vectors: Some( + HashMap::::from([( + new_vector.to_string(), + Document::new( + record.payload.get("text") + .and_then(|v| v.as_str()) + .map_or("", |v| v), + new_model, + ), + )]) + .into(), + ), + }) + .collect(); + + client + .update_vectors(UpdatePointVectorsBuilder::new(collection, point_vectors)) + .await?; + + if last_offset.is_none() { + break; + } + } + // @block-end re-embed-existing + + // @block-start search-with-old-vector + let old_vector_results = client + .query( + QueryPointsBuilder::new(collection) + .query(Query::new_nearest(Document::new("my query", old_model))) + .using(old_vector) + .limit(10), + ) + .await?; + // @block-end search-with-old-vector + + // @hide-start + _ = old_vector_results; + // @hide-end + + // @block-start search-with-new-vector + let new_vector_results = client + .query( + QueryPointsBuilder::new(collection) + .query(Query::new_nearest(Document::new("my query", new_model))) + .using(new_vector) + .limit(10), + ) + .await?; + // @block-end search-with-new-vector + + // @hide-start + _ = new_vector_results; + // @hide-end + + // @block-start delete-old-named-vector + client + .delete_vector_name(DeleteVectorNameRequestBuilder::new( + collection, + old_vector, + )) + .await?; + // @block-end delete-old-named-vector + Ok(()) } diff --git a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/typescript.ts b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/typescript.ts index ff90f44f3..87c50b4ea 100644 --- a/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/typescript.ts +++ b/qdrant-landing/content/documentation/headless/snippets/tutorial-model-migration/typescript.ts @@ -14,6 +14,10 @@ const OLD_COLLECTION = "old_collection"; const OLD_MODEL = "sentence-transformers/all-minilm-l6-v2"; const NEW_MODEL = "qdrant/clip-vit-b-32-text"; + +const COLLECTION = "my_collection"; +const OLD_VECTOR = "old-model"; +const NEW_VECTOR = "new-model"; // @hide-end // @block-start create-new-collection @@ -120,3 +124,92 @@ const resultsNew = await client.query(NEW_COLLECTION, { limit: 10, }); // @block-end search-new-collection + +// @block-start add-named-vector +await client.createVectorName(COLLECTION, NEW_VECTOR, { + dense: { + size: 512, // Size of the new embedding vectors + distance: "Cosine", // Similarity function for the new model + }, +}); +// @block-end add-named-vector + +// @block-start upsert-both-vectors +await client.upsert(COLLECTION, { + points: [ + { + id: 1, + vector: { + [OLD_VECTOR]: { + text: "Example document", + model: OLD_MODEL, + }, + [NEW_VECTOR]: { + text: "Example document", + model: NEW_MODEL, + }, + }, + payload: { text: "Example document" }, + }, + ], +}); +// @block-end upsert-both-vectors + +// @block-start re-embed-existing +let reEmbedLastOffset: number | string | undefined = undefined; +const reEmbedBatchSize = 100; +let reEmbedReachedEnd = false; + +while (!reEmbedReachedEnd) { + const reEmbedScrollResult = await client.scroll(COLLECTION, { + limit: reEmbedBatchSize, + offset: reEmbedLastOffset, + with_payload: true, + with_vector: false, + }); + + const records = reEmbedScrollResult.points; + reEmbedLastOffset = reEmbedScrollResult.next_page_offset as number | string | undefined; + + // Update only the new vector on each point; the old vector and payload are untouched + await client.updateVectors(COLLECTION, { + points: records.map((record) => ({ + id: record.id, + vector: { + [NEW_VECTOR]: { + text: ((record.payload?.text as string) ?? ""), + model: NEW_MODEL, + }, + }, + })), + }); + + reEmbedReachedEnd = reEmbedLastOffset == null; +} +// @block-end re-embed-existing + +// @block-start search-with-old-vector +const oldVectorResults = await client.query(COLLECTION, { + query: { + text: "my query", + model: OLD_MODEL, + }, + using: OLD_VECTOR, + limit: 10, +}); +// @block-end search-with-old-vector + +// @block-start search-with-new-vector +const newVectorResults = await client.query(COLLECTION, { + query: { + text: "my query", + model: NEW_MODEL, + }, + using: NEW_VECTOR, + limit: 10, +}); +// @block-end search-with-new-vector + +// @block-start delete-old-named-vector +await client.deleteVectorName(COLLECTION, OLD_VECTOR); +// @block-end delete-old-named-vector diff --git a/qdrant-landing/content/documentation/manage-data/collections.md b/qdrant-landing/content/documentation/manage-data/collections.md index ca2360309..02fd859b6 100644 --- a/qdrant-landing/content/documentation/manage-data/collections.md +++ b/qdrant-landing/content/documentation/manage-data/collections.md @@ -189,7 +189,7 @@ introduce huge overhead due to the rebuilding of the index. *Available as of v1.18.0* -Named vectors can be added to or removed from an existing collection without having to recreate the collection. +Named vectors can be added to or removed from an existing collection without having to recreate the collection. This is useful for [embedding model migration](/documentation/tutorials-operations/embedding-model-migration/): you can add a new vector for the new model, re-embed points in the background, and then remove the old vector when you're ready.