mirror of
https://github.com/qdrant/landing_page.git
synced 2026-10-09 12:58:31 +02:00
Add named vectors scenario to embedding migration tutorial (#2348)
* Initial commit * Edits * Update FAQ * Edits * Mention required versions
This commit is contained in:
+100
@@ -18,6 +18,10 @@ public class Snippet
|
||||
|
||||
string OLD_MODEL = "sentence-transformers/all-minilm-l6-v2";
|
||||
string NEW_MODEL = "qdrant/clip-vit-b-32-text";
|
||||
|
||||
string COLLECTION = "my_collection";
|
||||
string OLD_VECTOR = "old-model";
|
||||
string NEW_VECTOR = "new-model";
|
||||
// @hide-end
|
||||
|
||||
// @block-start create-new-collection
|
||||
@@ -150,5 +154,101 @@ public class Snippet
|
||||
limit: 10
|
||||
);
|
||||
// @block-end search-new-collection
|
||||
|
||||
// @block-start add-named-vector
|
||||
await client.CreateVectorNameAsync(new()
|
||||
{
|
||||
CollectionName = COLLECTION,
|
||||
VectorName = NEW_VECTOR,
|
||||
DenseConfig = new() { Size = 512, Distance = Distance.Cosine }
|
||||
});
|
||||
// @block-end add-named-vector
|
||||
|
||||
// @block-start upsert-both-vectors
|
||||
await client.UpsertAsync(
|
||||
collectionName: COLLECTION,
|
||||
points: new List<PointStruct>
|
||||
{
|
||||
new()
|
||||
{
|
||||
Id = 1,
|
||||
Vectors = new Dictionary<string, Vector>
|
||||
{
|
||||
[OLD_VECTOR] = new Document { Text = "Example document", Model = OLD_MODEL },
|
||||
[NEW_VECTOR] = new Document { Text = "Example document", Model = NEW_MODEL },
|
||||
},
|
||||
Payload = { ["text"] = "Example document" }
|
||||
}
|
||||
}
|
||||
);
|
||||
// @block-end upsert-both-vectors
|
||||
|
||||
// @block-start re-embed-existing
|
||||
PointId? reEmbedLastOffset = null;
|
||||
uint reEmbedBatchSize = 100;
|
||||
bool reEmbedReachedEnd = false;
|
||||
|
||||
while (!reEmbedReachedEnd)
|
||||
{
|
||||
var reEmbedScrollResult = await client.ScrollAsync(
|
||||
collectionName: COLLECTION,
|
||||
limit: reEmbedBatchSize,
|
||||
offset: reEmbedLastOffset,
|
||||
payloadSelector: true,
|
||||
vectorsSelector: false
|
||||
);
|
||||
|
||||
var reEmbedRecords = reEmbedScrollResult.Result;
|
||||
reEmbedLastOffset = reEmbedScrollResult.NextPageOffset;
|
||||
|
||||
var pointVectors = new List<PointVectors>();
|
||||
foreach (var record in reEmbedRecords)
|
||||
{
|
||||
var text = record.Payload.ContainsKey("text")
|
||||
? record.Payload["text"].StringValue
|
||||
: "";
|
||||
|
||||
// Update only the new vector on each point; the old vector and payload are untouched
|
||||
pointVectors.Add(new PointVectors
|
||||
{
|
||||
Id = record.Id,
|
||||
Vectors = new Dictionary<string, Vector>
|
||||
{
|
||||
[NEW_VECTOR] = new Document { Text = text, Model = NEW_MODEL }
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
await client.UpdateVectorsAsync(collectionName: COLLECTION, points: pointVectors);
|
||||
|
||||
reEmbedReachedEnd = (reEmbedLastOffset == null);
|
||||
}
|
||||
// @block-end re-embed-existing
|
||||
|
||||
// @block-start search-with-old-vector
|
||||
var oldVectorResults = await client.QueryAsync(
|
||||
collectionName: COLLECTION,
|
||||
query: new Document { Text = "my query", Model = OLD_MODEL },
|
||||
usingVector: OLD_VECTOR,
|
||||
limit: 10
|
||||
);
|
||||
// @block-end search-with-old-vector
|
||||
|
||||
// @block-start search-with-new-vector
|
||||
var newVectorResults = await client.QueryAsync(
|
||||
collectionName: COLLECTION,
|
||||
query: new Document { Text = "my query", Model = NEW_MODEL },
|
||||
usingVector: NEW_VECTOR,
|
||||
limit: 10
|
||||
);
|
||||
// @block-end search-with-new-vector
|
||||
|
||||
// @block-start delete-old-named-vector
|
||||
await client.DeleteVectorNameAsync(new()
|
||||
{
|
||||
CollectionName = COLLECTION,
|
||||
VectorName = OLD_VECTOR
|
||||
});
|
||||
// @block-end delete-old-named-vector
|
||||
}
|
||||
}
|
||||
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
```csharp
|
||||
await client.CreateVectorNameAsync(new()
|
||||
{
|
||||
CollectionName = COLLECTION,
|
||||
VectorName = NEW_VECTOR,
|
||||
DenseConfig = new() { Size = 512, Distance = Distance.Cosine }
|
||||
});
|
||||
```
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
```go
|
||||
client.CreateVectorName(context.Background(), &qdrant.CreateVectorNameRequest{
|
||||
CollectionName: COLLECTION,
|
||||
VectorName: NEW_VECTOR,
|
||||
VectorConfig: &qdrant.CreateVectorNameRequest_DenseConfig{
|
||||
DenseConfig: &qdrant.DenseVectorCreationConfig{
|
||||
Size: 512, // Size of the new embedding vectors
|
||||
Distance: qdrant.Distance_Cosine,
|
||||
},
|
||||
},
|
||||
})
|
||||
```
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
```java
|
||||
client
|
||||
.createVectorNameAsync(
|
||||
CreateVectorNameRequest.newBuilder()
|
||||
.setCollectionName(COLLECTION)
|
||||
.setVectorName(NEW_VECTOR)
|
||||
.setDenseConfig(
|
||||
DenseVectorCreationConfig.newBuilder()
|
||||
.setSize(512) // Size of the new embedding vectors
|
||||
.setDistance(Distance.Cosine) // Similarity function for the new model
|
||||
.build())
|
||||
.build())
|
||||
.get();
|
||||
```
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
```python
|
||||
client.create_vector_name(
|
||||
collection_name=COLLECTION,
|
||||
vector_name=NEW_VECTOR,
|
||||
vector_name_config=models.DenseVectorNameConfig(
|
||||
dense=models.DenseVectorConfig(
|
||||
size=512, # Size of the new embedding vectors
|
||||
distance=models.Distance.COSINE # Similarity function for the new model
|
||||
)
|
||||
),
|
||||
)
|
||||
```
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
```rust
|
||||
client
|
||||
.create_vector_name(
|
||||
CreateVectorNameRequestBuilder::new(
|
||||
collection,
|
||||
new_vector,
|
||||
DenseVectorCreationConfigBuilder::new(512, Distance::Cosine), // Size of the new embedding vectors
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
```
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
```typescript
|
||||
await client.createVectorName(COLLECTION, NEW_VECTOR, {
|
||||
dense: {
|
||||
size: 512, // Size of the new embedding vectors
|
||||
distance: "Cosine", // Similarity function for the new model
|
||||
},
|
||||
});
|
||||
```
|
||||
+84
@@ -120,4 +120,88 @@ results = await client.QueryAsync(
|
||||
},
|
||||
limit: 10
|
||||
);
|
||||
|
||||
await client.CreateVectorNameAsync(new()
|
||||
{
|
||||
CollectionName = COLLECTION,
|
||||
VectorName = NEW_VECTOR,
|
||||
DenseConfig = new() { Size = 512, Distance = Distance.Cosine }
|
||||
});
|
||||
|
||||
await client.UpsertAsync(
|
||||
collectionName: COLLECTION,
|
||||
points: new List<PointStruct>
|
||||
{
|
||||
new()
|
||||
{
|
||||
Id = 1,
|
||||
Vectors = new Dictionary<string, Vector>
|
||||
{
|
||||
[OLD_VECTOR] = new Document { Text = "Example document", Model = OLD_MODEL },
|
||||
[NEW_VECTOR] = new Document { Text = "Example document", Model = NEW_MODEL },
|
||||
},
|
||||
Payload = { ["text"] = "Example document" }
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
PointId? reEmbedLastOffset = null;
|
||||
uint reEmbedBatchSize = 100;
|
||||
bool reEmbedReachedEnd = false;
|
||||
|
||||
while (!reEmbedReachedEnd)
|
||||
{
|
||||
var reEmbedScrollResult = await client.ScrollAsync(
|
||||
collectionName: COLLECTION,
|
||||
limit: reEmbedBatchSize,
|
||||
offset: reEmbedLastOffset,
|
||||
payloadSelector: true,
|
||||
vectorsSelector: false
|
||||
);
|
||||
|
||||
var reEmbedRecords = reEmbedScrollResult.Result;
|
||||
reEmbedLastOffset = reEmbedScrollResult.NextPageOffset;
|
||||
|
||||
var pointVectors = new List<PointVectors>();
|
||||
foreach (var record in reEmbedRecords)
|
||||
{
|
||||
var text = record.Payload.ContainsKey("text")
|
||||
? record.Payload["text"].StringValue
|
||||
: "";
|
||||
|
||||
// Update only the new vector on each point; the old vector and payload are untouched
|
||||
pointVectors.Add(new PointVectors
|
||||
{
|
||||
Id = record.Id,
|
||||
Vectors = new Dictionary<string, Vector>
|
||||
{
|
||||
[NEW_VECTOR] = new Document { Text = text, Model = NEW_MODEL }
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
await client.UpdateVectorsAsync(collectionName: COLLECTION, points: pointVectors);
|
||||
|
||||
reEmbedReachedEnd = (reEmbedLastOffset == null);
|
||||
}
|
||||
|
||||
var oldVectorResults = await client.QueryAsync(
|
||||
collectionName: COLLECTION,
|
||||
query: new Document { Text = "my query", Model = OLD_MODEL },
|
||||
usingVector: OLD_VECTOR,
|
||||
limit: 10
|
||||
);
|
||||
|
||||
var newVectorResults = await client.QueryAsync(
|
||||
collectionName: COLLECTION,
|
||||
query: new Document { Text = "my query", Model = NEW_MODEL },
|
||||
usingVector: NEW_VECTOR,
|
||||
limit: 10
|
||||
);
|
||||
|
||||
await client.DeleteVectorNameAsync(new()
|
||||
{
|
||||
CollectionName = COLLECTION,
|
||||
VectorName = OLD_VECTOR
|
||||
});
|
||||
```
|
||||
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
```csharp
|
||||
await client.DeleteVectorNameAsync(new()
|
||||
{
|
||||
CollectionName = COLLECTION,
|
||||
VectorName = OLD_VECTOR
|
||||
});
|
||||
```
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
```go
|
||||
client.DeleteVectorName(context.Background(), &qdrant.DeleteVectorNameRequest{
|
||||
CollectionName: COLLECTION,
|
||||
VectorName: OLD_VECTOR,
|
||||
})
|
||||
```
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
```java
|
||||
client
|
||||
.deleteVectorNameAsync(
|
||||
DeleteVectorNameRequest.newBuilder()
|
||||
.setCollectionName(COLLECTION)
|
||||
.setVectorName(OLD_VECTOR)
|
||||
.build())
|
||||
.get();
|
||||
```
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
```python
|
||||
client.delete_vector_name(
|
||||
collection_name=COLLECTION,
|
||||
vector_name=OLD_VECTOR,
|
||||
)
|
||||
```
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
```rust
|
||||
client
|
||||
.delete_vector_name(DeleteVectorNameRequestBuilder::new(
|
||||
collection,
|
||||
old_vector,
|
||||
))
|
||||
.await?;
|
||||
```
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
```typescript
|
||||
await client.deleteVectorName(COLLECTION, OLD_VECTOR);
|
||||
```
|
||||
+98
@@ -111,4 +111,102 @@ results, err = client.Query(context.Background(), &qdrant.QueryPoints{
|
||||
}),
|
||||
Limit: qdrant.PtrOf(uint64(10)),
|
||||
})
|
||||
|
||||
client.CreateVectorName(context.Background(), &qdrant.CreateVectorNameRequest{
|
||||
CollectionName: COLLECTION,
|
||||
VectorName: NEW_VECTOR,
|
||||
VectorConfig: &qdrant.CreateVectorNameRequest_DenseConfig{
|
||||
DenseConfig: &qdrant.DenseVectorCreationConfig{
|
||||
Size: 512, // Size of the new embedding vectors
|
||||
Distance: qdrant.Distance_Cosine,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
client.Upsert(context.Background(), &qdrant.UpsertPoints{
|
||||
CollectionName: COLLECTION,
|
||||
Points: []*qdrant.PointStruct{
|
||||
{
|
||||
Id: qdrant.NewIDNum(1),
|
||||
Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{
|
||||
OLD_VECTOR: qdrant.NewVectorDocument(&qdrant.Document{
|
||||
Text: "Example document",
|
||||
Model: OLD_MODEL,
|
||||
}),
|
||||
NEW_VECTOR: qdrant.NewVectorDocument(&qdrant.Document{
|
||||
Text: "Example document",
|
||||
Model: NEW_MODEL,
|
||||
}),
|
||||
}),
|
||||
Payload: qdrant.NewValueMap(map[string]any{"text": "Example document"}),
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
var reEmbedLastOffset *qdrant.PointId
|
||||
reEmbedBatchSize := uint32(100)
|
||||
reEmbedReachedEnd := false
|
||||
|
||||
for !reEmbedReachedEnd {
|
||||
reEmbedScrollResult, err := client.Scroll(context.Background(), &qdrant.ScrollPoints{
|
||||
CollectionName: COLLECTION,
|
||||
Limit: qdrant.PtrOf(reEmbedBatchSize),
|
||||
Offset: reEmbedLastOffset,
|
||||
WithPayload: qdrant.NewWithPayload(true),
|
||||
WithVectors: qdrant.NewWithVectors(false),
|
||||
})
|
||||
|
||||
reEmbedRecords := reEmbedScrollResult
|
||||
|
||||
pointVectors := make([]*qdrant.PointVectors, len(reEmbedRecords))
|
||||
for idx, record := range reEmbedRecords {
|
||||
text := ""
|
||||
if val, ok := record.Payload["text"]; ok {
|
||||
text = val.GetStringValue()
|
||||
}
|
||||
|
||||
// Update only the new vector on each point; the old vector and payload are untouched
|
||||
pointVectors[idx] = &qdrant.PointVectors{
|
||||
Id: record.Id,
|
||||
Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{
|
||||
NEW_VECTOR: qdrant.NewVectorDocument(&qdrant.Document{
|
||||
Text: text,
|
||||
Model: NEW_MODEL,
|
||||
}),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
client.UpdateVectors(context.Background(), &qdrant.UpdatePointVectors{
|
||||
CollectionName: COLLECTION,
|
||||
Points: pointVectors,
|
||||
})
|
||||
|
||||
reEmbedReachedEnd = (reEmbedLastOffset == nil)
|
||||
}
|
||||
|
||||
oldVectorResults, err := client.Query(context.Background(), &qdrant.QueryPoints{
|
||||
CollectionName: COLLECTION,
|
||||
Query: qdrant.NewQueryDocument(&qdrant.Document{
|
||||
Text: "my query",
|
||||
Model: OLD_MODEL,
|
||||
}),
|
||||
Using: qdrant.PtrOf(OLD_VECTOR),
|
||||
Limit: qdrant.PtrOf(uint64(10)),
|
||||
})
|
||||
|
||||
newVectorResults, err := client.Query(context.Background(), &qdrant.QueryPoints{
|
||||
CollectionName: COLLECTION,
|
||||
Query: qdrant.NewQueryDocument(&qdrant.Document{
|
||||
Text: "my query",
|
||||
Model: NEW_MODEL,
|
||||
}),
|
||||
Using: qdrant.PtrOf(NEW_VECTOR),
|
||||
Limit: qdrant.PtrOf(uint64(10)),
|
||||
})
|
||||
|
||||
client.DeleteVectorName(context.Background(), &qdrant.DeleteVectorNameRequest{
|
||||
CollectionName: COLLECTION,
|
||||
VectorName: OLD_VECTOR,
|
||||
})
|
||||
```
|
||||
|
||||
+114
@@ -3,6 +3,7 @@ import static io.qdrant.client.PointIdFactory.id;
|
||||
import static io.qdrant.client.QueryFactory.nearest;
|
||||
import static io.qdrant.client.ValueFactory.value;
|
||||
import static io.qdrant.client.VectorFactory.vector;
|
||||
import static io.qdrant.client.VectorsFactory.namedVectors;
|
||||
import static io.qdrant.client.VectorsFactory.vectors;
|
||||
import io.qdrant.client.WithPayloadSelectorFactory;
|
||||
import io.qdrant.client.WithVectorsSelectorFactory;
|
||||
@@ -15,6 +16,10 @@ import io.qdrant.client.grpc.JsonWithInt.Value;
|
||||
import io.qdrant.client.grpc.Points.Document;
|
||||
import io.qdrant.client.grpc.Points.PointStruct;
|
||||
import io.qdrant.client.grpc.Points.QueryPoints;
|
||||
import io.qdrant.client.grpc.Points.CreateVectorNameRequest;
|
||||
import io.qdrant.client.grpc.Points.DeleteVectorNameRequest;
|
||||
import io.qdrant.client.grpc.Points.DenseVectorCreationConfig;
|
||||
import io.qdrant.client.grpc.Points.PointVectors;
|
||||
import io.qdrant.client.grpc.Points.UpsertPoints;
|
||||
import io.qdrant.client.grpc.Points.ScrollPoints;
|
||||
import io.qdrant.client.grpc.Points.UpdateMode;
|
||||
@@ -141,4 +146,113 @@ QueryPoints newRequest =
|
||||
.build();
|
||||
|
||||
results = client.queryAsync(newRequest).get();
|
||||
|
||||
client
|
||||
.createVectorNameAsync(
|
||||
CreateVectorNameRequest.newBuilder()
|
||||
.setCollectionName(COLLECTION)
|
||||
.setVectorName(NEW_VECTOR)
|
||||
.setDenseConfig(
|
||||
DenseVectorCreationConfig.newBuilder()
|
||||
.setSize(512) // Size of the new embedding vectors
|
||||
.setDistance(Distance.Cosine) // Similarity function for the new model
|
||||
.build())
|
||||
.build())
|
||||
.get();
|
||||
|
||||
client.upsertAsync(COLLECTION, List.of(
|
||||
PointStruct.newBuilder()
|
||||
.setId(id(1))
|
||||
.setVectors(
|
||||
namedVectors(
|
||||
Map.of(
|
||||
OLD_VECTOR, vector(
|
||||
Document.newBuilder()
|
||||
.setText("Example document")
|
||||
.setModel(OLD_MODEL)
|
||||
.build()),
|
||||
NEW_VECTOR, vector(
|
||||
Document.newBuilder()
|
||||
.setText("Example document")
|
||||
.setModel(NEW_MODEL)
|
||||
.build()))))
|
||||
.putAllPayload(Map.of("text", value("Example document")))
|
||||
.build())).get();
|
||||
|
||||
int reEmbedBatchSize = 100;
|
||||
boolean reEmbedReachedEnd = false;
|
||||
|
||||
var reEmbedScrollBuilder = ScrollPoints.newBuilder()
|
||||
.setCollectionName(COLLECTION)
|
||||
.setLimit(reEmbedBatchSize)
|
||||
.setWithPayload(WithPayloadSelectorFactory.enable(true))
|
||||
.setWithVectors(WithVectorsSelectorFactory.enable(false));
|
||||
|
||||
while (!reEmbedReachedEnd) {
|
||||
var reEmbedScrollResult = client.scrollAsync(reEmbedScrollBuilder.build()).get();
|
||||
var reEmbedRecords = reEmbedScrollResult.getResultList();
|
||||
|
||||
List<PointVectors> pointVectors = new ArrayList<>();
|
||||
for (var record : reEmbedRecords) {
|
||||
String text = record.getPayloadMap().containsKey("text")
|
||||
? record.getPayloadMap().get("text").getStringValue()
|
||||
: "";
|
||||
|
||||
// Update only the new vector on each point; the old vector and payload are untouched
|
||||
pointVectors.add(
|
||||
PointVectors.newBuilder()
|
||||
.setId(record.getId())
|
||||
.setVectors(
|
||||
namedVectors(
|
||||
Map.of(
|
||||
NEW_VECTOR, vector(
|
||||
Document.newBuilder()
|
||||
.setText(text)
|
||||
.setModel(NEW_MODEL)
|
||||
.build()))))
|
||||
.build());
|
||||
}
|
||||
|
||||
client.updateVectorsAsync(COLLECTION, pointVectors).get();
|
||||
|
||||
if (reEmbedScrollResult.hasNextPageOffset()) {
|
||||
reEmbedScrollBuilder.setOffset(reEmbedScrollResult.getNextPageOffset());
|
||||
} else {
|
||||
reEmbedReachedEnd = true;
|
||||
}
|
||||
}
|
||||
|
||||
var oldVectorResults = client.queryAsync(
|
||||
QueryPoints.newBuilder()
|
||||
.setCollectionName(COLLECTION)
|
||||
.setQuery(
|
||||
nearest(
|
||||
Document.newBuilder()
|
||||
.setText("my query")
|
||||
.setModel(OLD_MODEL)
|
||||
.build()))
|
||||
.setUsing(OLD_VECTOR)
|
||||
.setLimit(10)
|
||||
.build()).get();
|
||||
|
||||
var newVectorResults = client.queryAsync(
|
||||
QueryPoints.newBuilder()
|
||||
.setCollectionName(COLLECTION)
|
||||
.setQuery(
|
||||
nearest(
|
||||
Document.newBuilder()
|
||||
.setText("my query")
|
||||
.setModel(NEW_MODEL)
|
||||
.build()))
|
||||
.setUsing(NEW_VECTOR)
|
||||
.setLimit(10)
|
||||
.build()).get();
|
||||
|
||||
client
|
||||
.deleteVectorNameAsync(
|
||||
DeleteVectorNameRequest.newBuilder()
|
||||
.setCollectionName(COLLECTION)
|
||||
.setVectorName(OLD_VECTOR)
|
||||
.build())
|
||||
.get();
|
||||
```
|
||||
|
||||
+82
@@ -95,4 +95,86 @@ results = client.query_points(
|
||||
query=models.Document(text="my query", model=NEW_MODEL),
|
||||
limit=10,
|
||||
)
|
||||
|
||||
client.create_vector_name(
|
||||
collection_name=COLLECTION,
|
||||
vector_name=NEW_VECTOR,
|
||||
vector_name_config=models.DenseVectorNameConfig(
|
||||
dense=models.DenseVectorConfig(
|
||||
size=512, # Size of the new embedding vectors
|
||||
distance=models.Distance.COSINE # Similarity function for the new model
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
client.upsert(
|
||||
collection_name=COLLECTION,
|
||||
points=[
|
||||
models.PointStruct(
|
||||
id=1,
|
||||
vector={
|
||||
OLD_VECTOR: models.Document(
|
||||
text="Example document",
|
||||
model=OLD_MODEL,
|
||||
),
|
||||
NEW_VECTOR: models.Document(
|
||||
text="Example document",
|
||||
model=NEW_MODEL,
|
||||
),
|
||||
},
|
||||
payload={"text": "Example document"}
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
last_offset = None
|
||||
batch_size = 100
|
||||
reached_end = False
|
||||
|
||||
while not reached_end:
|
||||
records, last_offset = client.scroll(
|
||||
collection_name=COLLECTION,
|
||||
limit=batch_size,
|
||||
offset=last_offset,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
)
|
||||
|
||||
# Update only the new vector on each point; the old vector and payload are untouched
|
||||
client.update_vectors(
|
||||
collection_name=COLLECTION,
|
||||
points=[
|
||||
models.PointVectors(
|
||||
id=record.id,
|
||||
vector={
|
||||
NEW_VECTOR: models.Document(
|
||||
text=(record.payload or {}).get("text", ""),
|
||||
model=NEW_MODEL,
|
||||
)
|
||||
},
|
||||
)
|
||||
for record in records
|
||||
],
|
||||
)
|
||||
|
||||
reached_end = last_offset is None
|
||||
|
||||
results = client.query_points(
|
||||
collection_name=COLLECTION,
|
||||
query=models.Document(text="my query", model=OLD_MODEL),
|
||||
using=OLD_VECTOR,
|
||||
limit=10,
|
||||
)
|
||||
|
||||
results = client.query_points(
|
||||
collection_name=COLLECTION,
|
||||
query=models.Document(text="my query", model=NEW_MODEL),
|
||||
using=NEW_VECTOR,
|
||||
limit=10,
|
||||
)
|
||||
|
||||
client.delete_vector_name(
|
||||
collection_name=COLLECTION,
|
||||
vector_name=OLD_VECTOR,
|
||||
)
|
||||
```
|
||||
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
```csharp
|
||||
PointId? reEmbedLastOffset = null;
|
||||
uint reEmbedBatchSize = 100;
|
||||
bool reEmbedReachedEnd = false;
|
||||
|
||||
while (!reEmbedReachedEnd)
|
||||
{
|
||||
var reEmbedScrollResult = await client.ScrollAsync(
|
||||
collectionName: COLLECTION,
|
||||
limit: reEmbedBatchSize,
|
||||
offset: reEmbedLastOffset,
|
||||
payloadSelector: true,
|
||||
vectorsSelector: false
|
||||
);
|
||||
|
||||
var reEmbedRecords = reEmbedScrollResult.Result;
|
||||
reEmbedLastOffset = reEmbedScrollResult.NextPageOffset;
|
||||
|
||||
var pointVectors = new List<PointVectors>();
|
||||
foreach (var record in reEmbedRecords)
|
||||
{
|
||||
var text = record.Payload.ContainsKey("text")
|
||||
? record.Payload["text"].StringValue
|
||||
: "";
|
||||
|
||||
// Update only the new vector on each point; the old vector and payload are untouched
|
||||
pointVectors.Add(new PointVectors
|
||||
{
|
||||
Id = record.Id,
|
||||
Vectors = new Dictionary<string, Vector>
|
||||
{
|
||||
[NEW_VECTOR] = new Document { Text = text, Model = NEW_MODEL }
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
await client.UpdateVectorsAsync(collectionName: COLLECTION, points: pointVectors);
|
||||
|
||||
reEmbedReachedEnd = (reEmbedLastOffset == null);
|
||||
}
|
||||
```
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
```go
|
||||
var reEmbedLastOffset *qdrant.PointId
|
||||
reEmbedBatchSize := uint32(100)
|
||||
reEmbedReachedEnd := false
|
||||
|
||||
for !reEmbedReachedEnd {
|
||||
reEmbedScrollResult, err := client.Scroll(context.Background(), &qdrant.ScrollPoints{
|
||||
CollectionName: COLLECTION,
|
||||
Limit: qdrant.PtrOf(reEmbedBatchSize),
|
||||
Offset: reEmbedLastOffset,
|
||||
WithPayload: qdrant.NewWithPayload(true),
|
||||
WithVectors: qdrant.NewWithVectors(false),
|
||||
})
|
||||
|
||||
reEmbedRecords := reEmbedScrollResult
|
||||
|
||||
pointVectors := make([]*qdrant.PointVectors, len(reEmbedRecords))
|
||||
for idx, record := range reEmbedRecords {
|
||||
text := ""
|
||||
if val, ok := record.Payload["text"]; ok {
|
||||
text = val.GetStringValue()
|
||||
}
|
||||
|
||||
// Update only the new vector on each point; the old vector and payload are untouched
|
||||
pointVectors[idx] = &qdrant.PointVectors{
|
||||
Id: record.Id,
|
||||
Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{
|
||||
NEW_VECTOR: qdrant.NewVectorDocument(&qdrant.Document{
|
||||
Text: text,
|
||||
Model: NEW_MODEL,
|
||||
}),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
client.UpdateVectors(context.Background(), &qdrant.UpdatePointVectors{
|
||||
CollectionName: COLLECTION,
|
||||
Points: pointVectors,
|
||||
})
|
||||
|
||||
reEmbedReachedEnd = (reEmbedLastOffset == nil)
|
||||
}
|
||||
```
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
```java
|
||||
int reEmbedBatchSize = 100;
|
||||
boolean reEmbedReachedEnd = false;
|
||||
|
||||
var reEmbedScrollBuilder = ScrollPoints.newBuilder()
|
||||
.setCollectionName(COLLECTION)
|
||||
.setLimit(reEmbedBatchSize)
|
||||
.setWithPayload(WithPayloadSelectorFactory.enable(true))
|
||||
.setWithVectors(WithVectorsSelectorFactory.enable(false));
|
||||
|
||||
while (!reEmbedReachedEnd) {
|
||||
var reEmbedScrollResult = client.scrollAsync(reEmbedScrollBuilder.build()).get();
|
||||
var reEmbedRecords = reEmbedScrollResult.getResultList();
|
||||
|
||||
List<PointVectors> pointVectors = new ArrayList<>();
|
||||
for (var record : reEmbedRecords) {
|
||||
String text = record.getPayloadMap().containsKey("text")
|
||||
? record.getPayloadMap().get("text").getStringValue()
|
||||
: "";
|
||||
|
||||
// Update only the new vector on each point; the old vector and payload are untouched
|
||||
pointVectors.add(
|
||||
PointVectors.newBuilder()
|
||||
.setId(record.getId())
|
||||
.setVectors(
|
||||
namedVectors(
|
||||
Map.of(
|
||||
NEW_VECTOR, vector(
|
||||
Document.newBuilder()
|
||||
.setText(text)
|
||||
.setModel(NEW_MODEL)
|
||||
.build()))))
|
||||
.build());
|
||||
}
|
||||
|
||||
client.updateVectorsAsync(COLLECTION, pointVectors).get();
|
||||
|
||||
if (reEmbedScrollResult.hasNextPageOffset()) {
|
||||
reEmbedScrollBuilder.setOffset(reEmbedScrollResult.getNextPageOffset());
|
||||
} else {
|
||||
reEmbedReachedEnd = true;
|
||||
}
|
||||
}
|
||||
```
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
```python
|
||||
last_offset = None
|
||||
batch_size = 100
|
||||
reached_end = False
|
||||
|
||||
while not reached_end:
|
||||
records, last_offset = client.scroll(
|
||||
collection_name=COLLECTION,
|
||||
limit=batch_size,
|
||||
offset=last_offset,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
)
|
||||
|
||||
# Update only the new vector on each point; the old vector and payload are untouched
|
||||
client.update_vectors(
|
||||
collection_name=COLLECTION,
|
||||
points=[
|
||||
models.PointVectors(
|
||||
id=record.id,
|
||||
vector={
|
||||
NEW_VECTOR: models.Document(
|
||||
text=(record.payload or {}).get("text", ""),
|
||||
model=NEW_MODEL,
|
||||
)
|
||||
},
|
||||
)
|
||||
for record in records
|
||||
],
|
||||
)
|
||||
|
||||
reached_end = last_offset is None
|
||||
```
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
```rust
|
||||
let mut last_offset = None;
|
||||
let batch_size = 100;
|
||||
|
||||
loop {
|
||||
let mut scroll_builder = ScrollPointsBuilder::new(collection)
|
||||
.limit(batch_size)
|
||||
.with_payload(true)
|
||||
.with_vectors(false);
|
||||
|
||||
if let Some(offset) = last_offset {
|
||||
scroll_builder = scroll_builder.offset(offset);
|
||||
}
|
||||
|
||||
let scroll_result = client.scroll(scroll_builder).await?;
|
||||
let records = scroll_result.result;
|
||||
last_offset = scroll_result.next_page_offset;
|
||||
|
||||
// Update only the new vector on each point; the old vector and payload are untouched
|
||||
let point_vectors: Vec<PointVectors> = records
|
||||
.iter()
|
||||
.map(|record| PointVectors {
|
||||
id: record.id.clone(),
|
||||
vectors: Some(
|
||||
HashMap::<String, Document>::from([(
|
||||
new_vector.to_string(),
|
||||
Document::new(
|
||||
record.payload.get("text")
|
||||
.and_then(|v| v.as_str())
|
||||
.map_or("", |v| v),
|
||||
new_model,
|
||||
),
|
||||
)])
|
||||
.into(),
|
||||
),
|
||||
})
|
||||
.collect();
|
||||
|
||||
client
|
||||
.update_vectors(UpdatePointVectorsBuilder::new(collection, point_vectors))
|
||||
.await?;
|
||||
|
||||
if last_offset.is_none() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
```
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
```typescript
|
||||
let reEmbedLastOffset: number | string | undefined = undefined;
|
||||
const reEmbedBatchSize = 100;
|
||||
let reEmbedReachedEnd = false;
|
||||
|
||||
while (!reEmbedReachedEnd) {
|
||||
const reEmbedScrollResult = await client.scroll(COLLECTION, {
|
||||
limit: reEmbedBatchSize,
|
||||
offset: reEmbedLastOffset,
|
||||
with_payload: true,
|
||||
with_vector: false,
|
||||
});
|
||||
|
||||
const records = reEmbedScrollResult.points;
|
||||
reEmbedLastOffset = reEmbedScrollResult.next_page_offset as number | string | undefined;
|
||||
|
||||
// Update only the new vector on each point; the old vector and payload are untouched
|
||||
await client.updateVectors(COLLECTION, {
|
||||
points: records.map((record) => ({
|
||||
id: record.id,
|
||||
vector: {
|
||||
[NEW_VECTOR]: {
|
||||
text: ((record.payload?.text as string) ?? ""),
|
||||
model: NEW_MODEL,
|
||||
},
|
||||
},
|
||||
})),
|
||||
});
|
||||
|
||||
reEmbedReachedEnd = reEmbedLastOffset == null;
|
||||
}
|
||||
```
|
||||
+114
-2
@@ -1,7 +1,11 @@
|
||||
```rust
|
||||
use std::collections::HashMap;
|
||||
|
||||
use qdrant_client::qdrant::{
|
||||
CreateCollectionBuilder, Distance, Document, PointStruct, Query, QueryPointsBuilder,
|
||||
ScrollPointsBuilder, UpdateMode, UpsertPointsBuilder, VectorParamsBuilder,
|
||||
CreateCollectionBuilder, CreateVectorNameRequestBuilder, DeleteVectorNameRequestBuilder,
|
||||
DenseVectorCreationConfigBuilder, Distance, Document, NamedVectors, PointStruct, PointVectors,
|
||||
Query, QueryPointsBuilder, ScrollPointsBuilder, UpdateMode, UpdatePointVectorsBuilder,
|
||||
UpsertPointsBuilder, VectorParamsBuilder,
|
||||
};
|
||||
use qdrant_client::Qdrant;
|
||||
|
||||
@@ -107,4 +111,112 @@ let results = client
|
||||
.limit(10),
|
||||
)
|
||||
.await?;
|
||||
|
||||
client
|
||||
.create_vector_name(
|
||||
CreateVectorNameRequestBuilder::new(
|
||||
collection,
|
||||
new_vector,
|
||||
DenseVectorCreationConfigBuilder::new(512, Distance::Cosine), // Size of the new embedding vectors
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
|
||||
client
|
||||
.upsert_points(UpsertPointsBuilder::new(
|
||||
collection,
|
||||
vec![PointStruct::new(
|
||||
1,
|
||||
NamedVectors::default()
|
||||
.add_vector(
|
||||
old_vector,
|
||||
Document {
|
||||
text: "Example document".into(),
|
||||
model: old_model.into(),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.add_vector(
|
||||
new_vector,
|
||||
Document {
|
||||
text: "Example document".into(),
|
||||
model: new_model.into(),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
[("text", "Example document".into())],
|
||||
)],
|
||||
))
|
||||
.await?;
|
||||
|
||||
let mut last_offset = None;
|
||||
let batch_size = 100;
|
||||
|
||||
loop {
|
||||
let mut scroll_builder = ScrollPointsBuilder::new(collection)
|
||||
.limit(batch_size)
|
||||
.with_payload(true)
|
||||
.with_vectors(false);
|
||||
|
||||
if let Some(offset) = last_offset {
|
||||
scroll_builder = scroll_builder.offset(offset);
|
||||
}
|
||||
|
||||
let scroll_result = client.scroll(scroll_builder).await?;
|
||||
let records = scroll_result.result;
|
||||
last_offset = scroll_result.next_page_offset;
|
||||
|
||||
// Update only the new vector on each point; the old vector and payload are untouched
|
||||
let point_vectors: Vec<PointVectors> = records
|
||||
.iter()
|
||||
.map(|record| PointVectors {
|
||||
id: record.id.clone(),
|
||||
vectors: Some(
|
||||
HashMap::<String, Document>::from([(
|
||||
new_vector.to_string(),
|
||||
Document::new(
|
||||
record.payload.get("text")
|
||||
.and_then(|v| v.as_str())
|
||||
.map_or("", |v| v),
|
||||
new_model,
|
||||
),
|
||||
)])
|
||||
.into(),
|
||||
),
|
||||
})
|
||||
.collect();
|
||||
|
||||
client
|
||||
.update_vectors(UpdatePointVectorsBuilder::new(collection, point_vectors))
|
||||
.await?;
|
||||
|
||||
if last_offset.is_none() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let old_vector_results = client
|
||||
.query(
|
||||
QueryPointsBuilder::new(collection)
|
||||
.query(Query::new_nearest(Document::new("my query", old_model)))
|
||||
.using(old_vector)
|
||||
.limit(10),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let new_vector_results = client
|
||||
.query(
|
||||
QueryPointsBuilder::new(collection)
|
||||
.query(Query::new_nearest(Document::new("my query", new_model)))
|
||||
.using(new_vector)
|
||||
.limit(10),
|
||||
)
|
||||
.await?;
|
||||
|
||||
client
|
||||
.delete_vector_name(DeleteVectorNameRequestBuilder::new(
|
||||
collection,
|
||||
old_vector,
|
||||
))
|
||||
.await?;
|
||||
```
|
||||
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
```csharp
|
||||
var newVectorResults = await client.QueryAsync(
|
||||
collectionName: COLLECTION,
|
||||
query: new Document { Text = "my query", Model = NEW_MODEL },
|
||||
usingVector: NEW_VECTOR,
|
||||
limit: 10
|
||||
);
|
||||
```
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
```go
|
||||
newVectorResults, err := client.Query(context.Background(), &qdrant.QueryPoints{
|
||||
CollectionName: COLLECTION,
|
||||
Query: qdrant.NewQueryDocument(&qdrant.Document{
|
||||
Text: "my query",
|
||||
Model: NEW_MODEL,
|
||||
}),
|
||||
Using: qdrant.PtrOf(NEW_VECTOR),
|
||||
Limit: qdrant.PtrOf(uint64(10)),
|
||||
})
|
||||
```
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
```java
|
||||
var newVectorResults = client.queryAsync(
|
||||
QueryPoints.newBuilder()
|
||||
.setCollectionName(COLLECTION)
|
||||
.setQuery(
|
||||
nearest(
|
||||
Document.newBuilder()
|
||||
.setText("my query")
|
||||
.setModel(NEW_MODEL)
|
||||
.build()))
|
||||
.setUsing(NEW_VECTOR)
|
||||
.setLimit(10)
|
||||
.build()).get();
|
||||
```
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
```python
|
||||
results = client.query_points(
|
||||
collection_name=COLLECTION,
|
||||
query=models.Document(text="my query", model=NEW_MODEL),
|
||||
using=NEW_VECTOR,
|
||||
limit=10,
|
||||
)
|
||||
```
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
```rust
|
||||
let new_vector_results = client
|
||||
.query(
|
||||
QueryPointsBuilder::new(collection)
|
||||
.query(Query::new_nearest(Document::new("my query", new_model)))
|
||||
.using(new_vector)
|
||||
.limit(10),
|
||||
)
|
||||
.await?;
|
||||
```
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
```typescript
|
||||
const newVectorResults = await client.query(COLLECTION, {
|
||||
query: {
|
||||
text: "my query",
|
||||
model: NEW_MODEL,
|
||||
},
|
||||
using: NEW_VECTOR,
|
||||
limit: 10,
|
||||
});
|
||||
```
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
```csharp
|
||||
var oldVectorResults = await client.QueryAsync(
|
||||
collectionName: COLLECTION,
|
||||
query: new Document { Text = "my query", Model = OLD_MODEL },
|
||||
usingVector: OLD_VECTOR,
|
||||
limit: 10
|
||||
);
|
||||
```
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
```go
|
||||
oldVectorResults, err := client.Query(context.Background(), &qdrant.QueryPoints{
|
||||
CollectionName: COLLECTION,
|
||||
Query: qdrant.NewQueryDocument(&qdrant.Document{
|
||||
Text: "my query",
|
||||
Model: OLD_MODEL,
|
||||
}),
|
||||
Using: qdrant.PtrOf(OLD_VECTOR),
|
||||
Limit: qdrant.PtrOf(uint64(10)),
|
||||
})
|
||||
```
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
```java
|
||||
var oldVectorResults = client.queryAsync(
|
||||
QueryPoints.newBuilder()
|
||||
.setCollectionName(COLLECTION)
|
||||
.setQuery(
|
||||
nearest(
|
||||
Document.newBuilder()
|
||||
.setText("my query")
|
||||
.setModel(OLD_MODEL)
|
||||
.build()))
|
||||
.setUsing(OLD_VECTOR)
|
||||
.setLimit(10)
|
||||
.build()).get();
|
||||
```
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
```python
|
||||
results = client.query_points(
|
||||
collection_name=COLLECTION,
|
||||
query=models.Document(text="my query", model=OLD_MODEL),
|
||||
using=OLD_VECTOR,
|
||||
limit=10,
|
||||
)
|
||||
```
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
```rust
|
||||
let old_vector_results = client
|
||||
.query(
|
||||
QueryPointsBuilder::new(collection)
|
||||
.query(Query::new_nearest(Document::new("my query", old_model)))
|
||||
.using(old_vector)
|
||||
.limit(10),
|
||||
)
|
||||
.await?;
|
||||
```
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
```typescript
|
||||
const oldVectorResults = await client.query(COLLECTION, {
|
||||
query: {
|
||||
text: "my query",
|
||||
model: OLD_MODEL,
|
||||
},
|
||||
using: OLD_VECTOR,
|
||||
limit: 10,
|
||||
});
|
||||
```
|
||||
+77
@@ -93,4 +93,81 @@ const resultsNew = await client.query(NEW_COLLECTION, {
|
||||
},
|
||||
limit: 10,
|
||||
});
|
||||
|
||||
await client.createVectorName(COLLECTION, NEW_VECTOR, {
|
||||
dense: {
|
||||
size: 512, // Size of the new embedding vectors
|
||||
distance: "Cosine", // Similarity function for the new model
|
||||
},
|
||||
});
|
||||
|
||||
await client.upsert(COLLECTION, {
|
||||
points: [
|
||||
{
|
||||
id: 1,
|
||||
vector: {
|
||||
[OLD_VECTOR]: {
|
||||
text: "Example document",
|
||||
model: OLD_MODEL,
|
||||
},
|
||||
[NEW_VECTOR]: {
|
||||
text: "Example document",
|
||||
model: NEW_MODEL,
|
||||
},
|
||||
},
|
||||
payload: { text: "Example document" },
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
let reEmbedLastOffset: number | string | undefined = undefined;
|
||||
const reEmbedBatchSize = 100;
|
||||
let reEmbedReachedEnd = false;
|
||||
|
||||
while (!reEmbedReachedEnd) {
|
||||
const reEmbedScrollResult = await client.scroll(COLLECTION, {
|
||||
limit: reEmbedBatchSize,
|
||||
offset: reEmbedLastOffset,
|
||||
with_payload: true,
|
||||
with_vector: false,
|
||||
});
|
||||
|
||||
const records = reEmbedScrollResult.points;
|
||||
reEmbedLastOffset = reEmbedScrollResult.next_page_offset as number | string | undefined;
|
||||
|
||||
// Update only the new vector on each point; the old vector and payload are untouched
|
||||
await client.updateVectors(COLLECTION, {
|
||||
points: records.map((record) => ({
|
||||
id: record.id,
|
||||
vector: {
|
||||
[NEW_VECTOR]: {
|
||||
text: ((record.payload?.text as string) ?? ""),
|
||||
model: NEW_MODEL,
|
||||
},
|
||||
},
|
||||
})),
|
||||
});
|
||||
|
||||
reEmbedReachedEnd = reEmbedLastOffset == null;
|
||||
}
|
||||
|
||||
const oldVectorResults = await client.query(COLLECTION, {
|
||||
query: {
|
||||
text: "my query",
|
||||
model: OLD_MODEL,
|
||||
},
|
||||
using: OLD_VECTOR,
|
||||
limit: 10,
|
||||
});
|
||||
|
||||
const newVectorResults = await client.query(COLLECTION, {
|
||||
query: {
|
||||
text: "my query",
|
||||
model: NEW_MODEL,
|
||||
},
|
||||
using: NEW_VECTOR,
|
||||
limit: 10,
|
||||
});
|
||||
|
||||
await client.deleteVectorName(COLLECTION, OLD_VECTOR);
|
||||
```
|
||||
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
```csharp
|
||||
await client.UpsertAsync(
|
||||
collectionName: COLLECTION,
|
||||
points: new List<PointStruct>
|
||||
{
|
||||
new()
|
||||
{
|
||||
Id = 1,
|
||||
Vectors = new Dictionary<string, Vector>
|
||||
{
|
||||
[OLD_VECTOR] = new Document { Text = "Example document", Model = OLD_MODEL },
|
||||
[NEW_VECTOR] = new Document { Text = "Example document", Model = NEW_MODEL },
|
||||
},
|
||||
Payload = { ["text"] = "Example document" }
|
||||
}
|
||||
}
|
||||
);
|
||||
```
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
```go
|
||||
client.Upsert(context.Background(), &qdrant.UpsertPoints{
|
||||
CollectionName: COLLECTION,
|
||||
Points: []*qdrant.PointStruct{
|
||||
{
|
||||
Id: qdrant.NewIDNum(1),
|
||||
Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{
|
||||
OLD_VECTOR: qdrant.NewVectorDocument(&qdrant.Document{
|
||||
Text: "Example document",
|
||||
Model: OLD_MODEL,
|
||||
}),
|
||||
NEW_VECTOR: qdrant.NewVectorDocument(&qdrant.Document{
|
||||
Text: "Example document",
|
||||
Model: NEW_MODEL,
|
||||
}),
|
||||
}),
|
||||
Payload: qdrant.NewValueMap(map[string]any{"text": "Example document"}),
|
||||
},
|
||||
},
|
||||
})
|
||||
```
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
```java
|
||||
client.upsertAsync(COLLECTION, List.of(
|
||||
PointStruct.newBuilder()
|
||||
.setId(id(1))
|
||||
.setVectors(
|
||||
namedVectors(
|
||||
Map.of(
|
||||
OLD_VECTOR, vector(
|
||||
Document.newBuilder()
|
||||
.setText("Example document")
|
||||
.setModel(OLD_MODEL)
|
||||
.build()),
|
||||
NEW_VECTOR, vector(
|
||||
Document.newBuilder()
|
||||
.setText("Example document")
|
||||
.setModel(NEW_MODEL)
|
||||
.build()))))
|
||||
.putAllPayload(Map.of("text", value("Example document")))
|
||||
.build())).get();
|
||||
```
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
```python
|
||||
client.upsert(
|
||||
collection_name=COLLECTION,
|
||||
points=[
|
||||
models.PointStruct(
|
||||
id=1,
|
||||
vector={
|
||||
OLD_VECTOR: models.Document(
|
||||
text="Example document",
|
||||
model=OLD_MODEL,
|
||||
),
|
||||
NEW_VECTOR: models.Document(
|
||||
text="Example document",
|
||||
model=NEW_MODEL,
|
||||
),
|
||||
},
|
||||
payload={"text": "Example document"}
|
||||
)
|
||||
]
|
||||
)
|
||||
```
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
```rust
|
||||
client
|
||||
.upsert_points(UpsertPointsBuilder::new(
|
||||
collection,
|
||||
vec![PointStruct::new(
|
||||
1,
|
||||
NamedVectors::default()
|
||||
.add_vector(
|
||||
old_vector,
|
||||
Document {
|
||||
text: "Example document".into(),
|
||||
model: old_model.into(),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.add_vector(
|
||||
new_vector,
|
||||
Document {
|
||||
text: "Example document".into(),
|
||||
model: new_model.into(),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
[("text", "Example document".into())],
|
||||
)],
|
||||
))
|
||||
.await?;
|
||||
```
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
```typescript
|
||||
await client.upsert(COLLECTION, {
|
||||
points: [
|
||||
{
|
||||
id: 1,
|
||||
vector: {
|
||||
[OLD_VECTOR]: {
|
||||
text: "Example document",
|
||||
model: OLD_MODEL,
|
||||
},
|
||||
[NEW_VECTOR]: {
|
||||
text: "Example document",
|
||||
model: NEW_MODEL,
|
||||
},
|
||||
},
|
||||
payload: { text: "Example document" },
|
||||
},
|
||||
],
|
||||
});
|
||||
```
|
||||
+135
@@ -23,6 +23,10 @@ func Main() {
|
||||
|
||||
OLD_MODEL := "sentence-transformers/all-minilm-l6-v2"
|
||||
NEW_MODEL := "qdrant/clip-vit-b-32-text"
|
||||
|
||||
COLLECTION := "my_collection"
|
||||
OLD_VECTOR := "old-model"
|
||||
NEW_VECTOR := "new-model"
|
||||
// @hide-end
|
||||
|
||||
// @block-start create-new-collection
|
||||
@@ -164,4 +168,135 @@ func Main() {
|
||||
}
|
||||
_ = results
|
||||
// @hide-end
|
||||
|
||||
// @block-start add-named-vector
|
||||
client.CreateVectorName(context.Background(), &qdrant.CreateVectorNameRequest{
|
||||
CollectionName: COLLECTION,
|
||||
VectorName: NEW_VECTOR,
|
||||
VectorConfig: &qdrant.CreateVectorNameRequest_DenseConfig{
|
||||
DenseConfig: &qdrant.DenseVectorCreationConfig{
|
||||
Size: 512, // Size of the new embedding vectors
|
||||
Distance: qdrant.Distance_Cosine,
|
||||
},
|
||||
},
|
||||
})
|
||||
// @block-end add-named-vector
|
||||
|
||||
// @block-start upsert-both-vectors
|
||||
client.Upsert(context.Background(), &qdrant.UpsertPoints{
|
||||
CollectionName: COLLECTION,
|
||||
Points: []*qdrant.PointStruct{
|
||||
{
|
||||
Id: qdrant.NewIDNum(1),
|
||||
Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{
|
||||
OLD_VECTOR: qdrant.NewVectorDocument(&qdrant.Document{
|
||||
Text: "Example document",
|
||||
Model: OLD_MODEL,
|
||||
}),
|
||||
NEW_VECTOR: qdrant.NewVectorDocument(&qdrant.Document{
|
||||
Text: "Example document",
|
||||
Model: NEW_MODEL,
|
||||
}),
|
||||
}),
|
||||
Payload: qdrant.NewValueMap(map[string]any{"text": "Example document"}),
|
||||
},
|
||||
},
|
||||
})
|
||||
// @block-end upsert-both-vectors
|
||||
|
||||
// @block-start re-embed-existing
|
||||
var reEmbedLastOffset *qdrant.PointId
|
||||
reEmbedBatchSize := uint32(100)
|
||||
reEmbedReachedEnd := false
|
||||
|
||||
for !reEmbedReachedEnd {
|
||||
reEmbedScrollResult, err := client.Scroll(context.Background(), &qdrant.ScrollPoints{
|
||||
CollectionName: COLLECTION,
|
||||
Limit: qdrant.PtrOf(reEmbedBatchSize),
|
||||
Offset: reEmbedLastOffset,
|
||||
WithPayload: qdrant.NewWithPayload(true),
|
||||
WithVectors: qdrant.NewWithVectors(false),
|
||||
})
|
||||
|
||||
// @hide-start
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// @hide-end
|
||||
|
||||
reEmbedRecords := reEmbedScrollResult
|
||||
reEmbedLastOffset = reEmbedScrollResult[len(reEmbedScrollResult)-1].Id // @hide
|
||||
|
||||
pointVectors := make([]*qdrant.PointVectors, len(reEmbedRecords))
|
||||
for idx, record := range reEmbedRecords {
|
||||
text := ""
|
||||
if val, ok := record.Payload["text"]; ok {
|
||||
text = val.GetStringValue()
|
||||
}
|
||||
|
||||
// Update only the new vector on each point; the old vector and payload are untouched
|
||||
pointVectors[idx] = &qdrant.PointVectors{
|
||||
Id: record.Id,
|
||||
Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{
|
||||
NEW_VECTOR: qdrant.NewVectorDocument(&qdrant.Document{
|
||||
Text: text,
|
||||
Model: NEW_MODEL,
|
||||
}),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
client.UpdateVectors(context.Background(), &qdrant.UpdatePointVectors{
|
||||
CollectionName: COLLECTION,
|
||||
Points: pointVectors,
|
||||
})
|
||||
|
||||
reEmbedReachedEnd = (reEmbedLastOffset == nil)
|
||||
}
|
||||
// @block-end re-embed-existing
|
||||
|
||||
// @block-start search-with-old-vector
|
||||
oldVectorResults, err := client.Query(context.Background(), &qdrant.QueryPoints{
|
||||
CollectionName: COLLECTION,
|
||||
Query: qdrant.NewQueryDocument(&qdrant.Document{
|
||||
Text: "my query",
|
||||
Model: OLD_MODEL,
|
||||
}),
|
||||
Using: qdrant.PtrOf(OLD_VECTOR),
|
||||
Limit: qdrant.PtrOf(uint64(10)),
|
||||
})
|
||||
// @block-end search-with-old-vector
|
||||
|
||||
// @hide-start
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
_ = oldVectorResults
|
||||
// @hide-end
|
||||
|
||||
// @block-start search-with-new-vector
|
||||
newVectorResults, err := client.Query(context.Background(), &qdrant.QueryPoints{
|
||||
CollectionName: COLLECTION,
|
||||
Query: qdrant.NewQueryDocument(&qdrant.Document{
|
||||
Text: "my query",
|
||||
Model: NEW_MODEL,
|
||||
}),
|
||||
Using: qdrant.PtrOf(NEW_VECTOR),
|
||||
Limit: qdrant.PtrOf(uint64(10)),
|
||||
})
|
||||
// @block-end search-with-new-vector
|
||||
|
||||
// @hide-start
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
_ = newVectorResults
|
||||
// @hide-end
|
||||
|
||||
// @block-start delete-old-named-vector
|
||||
client.DeleteVectorName(context.Background(), &qdrant.DeleteVectorNameRequest{
|
||||
CollectionName: COLLECTION,
|
||||
VectorName: OLD_VECTOR,
|
||||
})
|
||||
// @block-end delete-old-named-vector
|
||||
}
|
||||
|
||||
+130
@@ -4,6 +4,7 @@ import static io.qdrant.client.PointIdFactory.id;
|
||||
import static io.qdrant.client.QueryFactory.nearest;
|
||||
import static io.qdrant.client.ValueFactory.value;
|
||||
import static io.qdrant.client.VectorFactory.vector;
|
||||
import static io.qdrant.client.VectorsFactory.namedVectors;
|
||||
import static io.qdrant.client.VectorsFactory.vectors;
|
||||
import io.qdrant.client.WithPayloadSelectorFactory;
|
||||
import io.qdrant.client.WithVectorsSelectorFactory;
|
||||
@@ -16,6 +17,10 @@ import io.qdrant.client.grpc.JsonWithInt.Value;
|
||||
import io.qdrant.client.grpc.Points.Document;
|
||||
import io.qdrant.client.grpc.Points.PointStruct;
|
||||
import io.qdrant.client.grpc.Points.QueryPoints;
|
||||
import io.qdrant.client.grpc.Points.CreateVectorNameRequest;
|
||||
import io.qdrant.client.grpc.Points.DeleteVectorNameRequest;
|
||||
import io.qdrant.client.grpc.Points.DenseVectorCreationConfig;
|
||||
import io.qdrant.client.grpc.Points.PointVectors;
|
||||
import io.qdrant.client.grpc.Points.UpsertPoints;
|
||||
import io.qdrant.client.grpc.Points.ScrollPoints;
|
||||
import io.qdrant.client.grpc.Points.UpdateMode;
|
||||
@@ -41,6 +46,10 @@ public class Snippet {
|
||||
|
||||
String OLD_MODEL = "sentence-transformers/all-minilm-l6-v2";
|
||||
String NEW_MODEL = "qdrant/clip-vit-b-32-text";
|
||||
|
||||
String COLLECTION = "my_collection";
|
||||
String OLD_VECTOR = "old-model";
|
||||
String NEW_VECTOR = "new-model";
|
||||
// @hide-end
|
||||
|
||||
// @block-start create-new-collection
|
||||
@@ -174,6 +183,127 @@ public class Snippet {
|
||||
|
||||
results = client.queryAsync(newRequest).get();
|
||||
// @block-end search-new-collection
|
||||
|
||||
// @block-start add-named-vector
|
||||
client
|
||||
.createVectorNameAsync(
|
||||
CreateVectorNameRequest.newBuilder()
|
||||
.setCollectionName(COLLECTION)
|
||||
.setVectorName(NEW_VECTOR)
|
||||
.setDenseConfig(
|
||||
DenseVectorCreationConfig.newBuilder()
|
||||
.setSize(512) // Size of the new embedding vectors
|
||||
.setDistance(Distance.Cosine) // Similarity function for the new model
|
||||
.build())
|
||||
.build())
|
||||
.get();
|
||||
// @block-end add-named-vector
|
||||
|
||||
// @block-start upsert-both-vectors
|
||||
client.upsertAsync(COLLECTION, List.of(
|
||||
PointStruct.newBuilder()
|
||||
.setId(id(1))
|
||||
.setVectors(
|
||||
namedVectors(
|
||||
Map.of(
|
||||
OLD_VECTOR, vector(
|
||||
Document.newBuilder()
|
||||
.setText("Example document")
|
||||
.setModel(OLD_MODEL)
|
||||
.build()),
|
||||
NEW_VECTOR, vector(
|
||||
Document.newBuilder()
|
||||
.setText("Example document")
|
||||
.setModel(NEW_MODEL)
|
||||
.build()))))
|
||||
.putAllPayload(Map.of("text", value("Example document")))
|
||||
.build())).get();
|
||||
// @block-end upsert-both-vectors
|
||||
|
||||
// @block-start re-embed-existing
|
||||
int reEmbedBatchSize = 100;
|
||||
boolean reEmbedReachedEnd = false;
|
||||
|
||||
var reEmbedScrollBuilder = ScrollPoints.newBuilder()
|
||||
.setCollectionName(COLLECTION)
|
||||
.setLimit(reEmbedBatchSize)
|
||||
.setWithPayload(WithPayloadSelectorFactory.enable(true))
|
||||
.setWithVectors(WithVectorsSelectorFactory.enable(false));
|
||||
|
||||
while (!reEmbedReachedEnd) {
|
||||
var reEmbedScrollResult = client.scrollAsync(reEmbedScrollBuilder.build()).get();
|
||||
var reEmbedRecords = reEmbedScrollResult.getResultList();
|
||||
|
||||
List<PointVectors> pointVectors = new ArrayList<>();
|
||||
for (var record : reEmbedRecords) {
|
||||
String text = record.getPayloadMap().containsKey("text")
|
||||
? record.getPayloadMap().get("text").getStringValue()
|
||||
: "";
|
||||
|
||||
// Update only the new vector on each point; the old vector and payload are untouched
|
||||
pointVectors.add(
|
||||
PointVectors.newBuilder()
|
||||
.setId(record.getId())
|
||||
.setVectors(
|
||||
namedVectors(
|
||||
Map.of(
|
||||
NEW_VECTOR, vector(
|
||||
Document.newBuilder()
|
||||
.setText(text)
|
||||
.setModel(NEW_MODEL)
|
||||
.build()))))
|
||||
.build());
|
||||
}
|
||||
|
||||
client.updateVectorsAsync(COLLECTION, pointVectors).get();
|
||||
|
||||
if (reEmbedScrollResult.hasNextPageOffset()) {
|
||||
reEmbedScrollBuilder.setOffset(reEmbedScrollResult.getNextPageOffset());
|
||||
} else {
|
||||
reEmbedReachedEnd = true;
|
||||
}
|
||||
}
|
||||
// @block-end re-embed-existing
|
||||
|
||||
// @block-start search-with-old-vector
|
||||
var oldVectorResults = client.queryAsync(
|
||||
QueryPoints.newBuilder()
|
||||
.setCollectionName(COLLECTION)
|
||||
.setQuery(
|
||||
nearest(
|
||||
Document.newBuilder()
|
||||
.setText("my query")
|
||||
.setModel(OLD_MODEL)
|
||||
.build()))
|
||||
.setUsing(OLD_VECTOR)
|
||||
.setLimit(10)
|
||||
.build()).get();
|
||||
// @block-end search-with-old-vector
|
||||
|
||||
// @block-start search-with-new-vector
|
||||
var newVectorResults = client.queryAsync(
|
||||
QueryPoints.newBuilder()
|
||||
.setCollectionName(COLLECTION)
|
||||
.setQuery(
|
||||
nearest(
|
||||
Document.newBuilder()
|
||||
.setText("my query")
|
||||
.setModel(NEW_MODEL)
|
||||
.build()))
|
||||
.setUsing(NEW_VECTOR)
|
||||
.setLimit(10)
|
||||
.build()).get();
|
||||
// @block-end search-with-new-vector
|
||||
|
||||
// @block-start delete-old-named-vector
|
||||
client
|
||||
.deleteVectorNameAsync(
|
||||
DeleteVectorNameRequest.newBuilder()
|
||||
.setCollectionName(COLLECTION)
|
||||
.setVectorName(OLD_VECTOR)
|
||||
.build())
|
||||
.get();
|
||||
// @block-end delete-old-named-vector
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+98
@@ -11,6 +11,10 @@ OLD_COLLECTION="old_collection"
|
||||
|
||||
OLD_MODEL="sentence-transformers/all-minilm-l6-v2"
|
||||
NEW_MODEL="qdrant/clip-vit-b-32-text"
|
||||
|
||||
COLLECTION="my_collection"
|
||||
OLD_VECTOR="old-model"
|
||||
NEW_VECTOR="new-model"
|
||||
# @hide-end
|
||||
|
||||
# @block-start create-new-collection
|
||||
@@ -119,3 +123,97 @@ results = client.query_points(
|
||||
limit=10,
|
||||
)
|
||||
# @block-end search-new-collection
|
||||
|
||||
# @block-start add-named-vector
|
||||
client.create_vector_name(
|
||||
collection_name=COLLECTION,
|
||||
vector_name=NEW_VECTOR,
|
||||
vector_name_config=models.DenseVectorNameConfig(
|
||||
dense=models.DenseVectorConfig(
|
||||
size=512, # Size of the new embedding vectors
|
||||
distance=models.Distance.COSINE # Similarity function for the new model
|
||||
)
|
||||
),
|
||||
)
|
||||
# @block-end add-named-vector
|
||||
|
||||
# @block-start upsert-both-vectors
|
||||
client.upsert(
|
||||
collection_name=COLLECTION,
|
||||
points=[
|
||||
models.PointStruct(
|
||||
id=1,
|
||||
vector={
|
||||
OLD_VECTOR: models.Document(
|
||||
text="Example document",
|
||||
model=OLD_MODEL,
|
||||
),
|
||||
NEW_VECTOR: models.Document(
|
||||
text="Example document",
|
||||
model=NEW_MODEL,
|
||||
),
|
||||
},
|
||||
payload={"text": "Example document"}
|
||||
)
|
||||
]
|
||||
)
|
||||
# @block-end upsert-both-vectors
|
||||
|
||||
# @block-start re-embed-existing
|
||||
last_offset = None
|
||||
batch_size = 100
|
||||
reached_end = False
|
||||
|
||||
while not reached_end:
|
||||
records, last_offset = client.scroll(
|
||||
collection_name=COLLECTION,
|
||||
limit=batch_size,
|
||||
offset=last_offset,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
)
|
||||
|
||||
# Update only the new vector on each point; the old vector and payload are untouched
|
||||
client.update_vectors(
|
||||
collection_name=COLLECTION,
|
||||
points=[
|
||||
models.PointVectors(
|
||||
id=record.id,
|
||||
vector={
|
||||
NEW_VECTOR: models.Document(
|
||||
text=(record.payload or {}).get("text", ""),
|
||||
model=NEW_MODEL,
|
||||
)
|
||||
},
|
||||
)
|
||||
for record in records
|
||||
],
|
||||
)
|
||||
|
||||
reached_end = last_offset is None
|
||||
# @block-end re-embed-existing
|
||||
|
||||
# @block-start search-with-old-vector
|
||||
results = client.query_points(
|
||||
collection_name=COLLECTION,
|
||||
query=models.Document(text="my query", model=OLD_MODEL),
|
||||
using=OLD_VECTOR,
|
||||
limit=10,
|
||||
)
|
||||
# @block-end search-with-old-vector
|
||||
|
||||
# @block-start search-with-new-vector
|
||||
results = client.query_points(
|
||||
collection_name=COLLECTION,
|
||||
query=models.Document(text="my query", model=NEW_MODEL),
|
||||
using=NEW_VECTOR,
|
||||
limit=10,
|
||||
)
|
||||
# @block-end search-with-new-vector
|
||||
|
||||
# @block-start delete-old-named-vector
|
||||
client.delete_vector_name(
|
||||
collection_name=COLLECTION,
|
||||
vector_name=OLD_VECTOR,
|
||||
)
|
||||
# @block-end delete-old-named-vector
|
||||
|
||||
+138
-2
@@ -1,6 +1,10 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use qdrant_client::qdrant::{
|
||||
CreateCollectionBuilder, Distance, Document, PointStruct, Query, QueryPointsBuilder,
|
||||
ScrollPointsBuilder, UpdateMode, UpsertPointsBuilder, VectorParamsBuilder,
|
||||
CreateCollectionBuilder, CreateVectorNameRequestBuilder, DeleteVectorNameRequestBuilder,
|
||||
DenseVectorCreationConfigBuilder, Distance, Document, NamedVectors, PointStruct, PointVectors,
|
||||
Query, QueryPointsBuilder, ScrollPointsBuilder, UpdateMode, UpdatePointVectorsBuilder,
|
||||
UpsertPointsBuilder, VectorParamsBuilder,
|
||||
};
|
||||
use qdrant_client::Qdrant;
|
||||
|
||||
@@ -18,6 +22,10 @@ pub async fn main() -> anyhow::Result<()> {
|
||||
|
||||
let old_model = "sentence-transformers/all-minilm-l6-v2";
|
||||
let new_model = "qdrant/clip-vit-b-32-text";
|
||||
|
||||
let collection = "my_collection";
|
||||
let old_vector = "old-model";
|
||||
let new_vector = "new-model";
|
||||
// @hide-end
|
||||
|
||||
// @block-start create-new-collection
|
||||
@@ -143,5 +151,133 @@ pub async fn main() -> anyhow::Result<()> {
|
||||
_ = results;
|
||||
// @hide-end
|
||||
|
||||
// @block-start add-named-vector
|
||||
client
|
||||
.create_vector_name(
|
||||
CreateVectorNameRequestBuilder::new(
|
||||
collection,
|
||||
new_vector,
|
||||
DenseVectorCreationConfigBuilder::new(512, Distance::Cosine), // Size of the new embedding vectors
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
// @block-end add-named-vector
|
||||
|
||||
// @block-start upsert-both-vectors
|
||||
client
|
||||
.upsert_points(UpsertPointsBuilder::new(
|
||||
collection,
|
||||
vec![PointStruct::new(
|
||||
1,
|
||||
NamedVectors::default()
|
||||
.add_vector(
|
||||
old_vector,
|
||||
Document {
|
||||
text: "Example document".into(),
|
||||
model: old_model.into(),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.add_vector(
|
||||
new_vector,
|
||||
Document {
|
||||
text: "Example document".into(),
|
||||
model: new_model.into(),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
[("text", "Example document".into())],
|
||||
)],
|
||||
))
|
||||
.await?;
|
||||
// @block-end upsert-both-vectors
|
||||
|
||||
// @block-start re-embed-existing
|
||||
let mut last_offset = None;
|
||||
let batch_size = 100;
|
||||
|
||||
loop {
|
||||
let mut scroll_builder = ScrollPointsBuilder::new(collection)
|
||||
.limit(batch_size)
|
||||
.with_payload(true)
|
||||
.with_vectors(false);
|
||||
|
||||
if let Some(offset) = last_offset {
|
||||
scroll_builder = scroll_builder.offset(offset);
|
||||
}
|
||||
|
||||
let scroll_result = client.scroll(scroll_builder).await?;
|
||||
let records = scroll_result.result;
|
||||
last_offset = scroll_result.next_page_offset;
|
||||
|
||||
// Update only the new vector on each point; the old vector and payload are untouched
|
||||
let point_vectors: Vec<PointVectors> = records
|
||||
.iter()
|
||||
.map(|record| PointVectors {
|
||||
id: record.id.clone(),
|
||||
vectors: Some(
|
||||
HashMap::<String, Document>::from([(
|
||||
new_vector.to_string(),
|
||||
Document::new(
|
||||
record.payload.get("text")
|
||||
.and_then(|v| v.as_str())
|
||||
.map_or("", |v| v),
|
||||
new_model,
|
||||
),
|
||||
)])
|
||||
.into(),
|
||||
),
|
||||
})
|
||||
.collect();
|
||||
|
||||
client
|
||||
.update_vectors(UpdatePointVectorsBuilder::new(collection, point_vectors))
|
||||
.await?;
|
||||
|
||||
if last_offset.is_none() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
// @block-end re-embed-existing
|
||||
|
||||
// @block-start search-with-old-vector
|
||||
let old_vector_results = client
|
||||
.query(
|
||||
QueryPointsBuilder::new(collection)
|
||||
.query(Query::new_nearest(Document::new("my query", old_model)))
|
||||
.using(old_vector)
|
||||
.limit(10),
|
||||
)
|
||||
.await?;
|
||||
// @block-end search-with-old-vector
|
||||
|
||||
// @hide-start
|
||||
_ = old_vector_results;
|
||||
// @hide-end
|
||||
|
||||
// @block-start search-with-new-vector
|
||||
let new_vector_results = client
|
||||
.query(
|
||||
QueryPointsBuilder::new(collection)
|
||||
.query(Query::new_nearest(Document::new("my query", new_model)))
|
||||
.using(new_vector)
|
||||
.limit(10),
|
||||
)
|
||||
.await?;
|
||||
// @block-end search-with-new-vector
|
||||
|
||||
// @hide-start
|
||||
_ = new_vector_results;
|
||||
// @hide-end
|
||||
|
||||
// @block-start delete-old-named-vector
|
||||
client
|
||||
.delete_vector_name(DeleteVectorNameRequestBuilder::new(
|
||||
collection,
|
||||
old_vector,
|
||||
))
|
||||
.await?;
|
||||
// @block-end delete-old-named-vector
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
+93
@@ -14,6 +14,10 @@ const OLD_COLLECTION = "old_collection";
|
||||
|
||||
const OLD_MODEL = "sentence-transformers/all-minilm-l6-v2";
|
||||
const NEW_MODEL = "qdrant/clip-vit-b-32-text";
|
||||
|
||||
const COLLECTION = "my_collection";
|
||||
const OLD_VECTOR = "old-model";
|
||||
const NEW_VECTOR = "new-model";
|
||||
// @hide-end
|
||||
|
||||
// @block-start create-new-collection
|
||||
@@ -120,3 +124,92 @@ const resultsNew = await client.query(NEW_COLLECTION, {
|
||||
limit: 10,
|
||||
});
|
||||
// @block-end search-new-collection
|
||||
|
||||
// @block-start add-named-vector
|
||||
await client.createVectorName(COLLECTION, NEW_VECTOR, {
|
||||
dense: {
|
||||
size: 512, // Size of the new embedding vectors
|
||||
distance: "Cosine", // Similarity function for the new model
|
||||
},
|
||||
});
|
||||
// @block-end add-named-vector
|
||||
|
||||
// @block-start upsert-both-vectors
|
||||
await client.upsert(COLLECTION, {
|
||||
points: [
|
||||
{
|
||||
id: 1,
|
||||
vector: {
|
||||
[OLD_VECTOR]: {
|
||||
text: "Example document",
|
||||
model: OLD_MODEL,
|
||||
},
|
||||
[NEW_VECTOR]: {
|
||||
text: "Example document",
|
||||
model: NEW_MODEL,
|
||||
},
|
||||
},
|
||||
payload: { text: "Example document" },
|
||||
},
|
||||
],
|
||||
});
|
||||
// @block-end upsert-both-vectors
|
||||
|
||||
// @block-start re-embed-existing
|
||||
let reEmbedLastOffset: number | string | undefined = undefined;
|
||||
const reEmbedBatchSize = 100;
|
||||
let reEmbedReachedEnd = false;
|
||||
|
||||
while (!reEmbedReachedEnd) {
|
||||
const reEmbedScrollResult = await client.scroll(COLLECTION, {
|
||||
limit: reEmbedBatchSize,
|
||||
offset: reEmbedLastOffset,
|
||||
with_payload: true,
|
||||
with_vector: false,
|
||||
});
|
||||
|
||||
const records = reEmbedScrollResult.points;
|
||||
reEmbedLastOffset = reEmbedScrollResult.next_page_offset as number | string | undefined;
|
||||
|
||||
// Update only the new vector on each point; the old vector and payload are untouched
|
||||
await client.updateVectors(COLLECTION, {
|
||||
points: records.map((record) => ({
|
||||
id: record.id,
|
||||
vector: {
|
||||
[NEW_VECTOR]: {
|
||||
text: ((record.payload?.text as string) ?? ""),
|
||||
model: NEW_MODEL,
|
||||
},
|
||||
},
|
||||
})),
|
||||
});
|
||||
|
||||
reEmbedReachedEnd = reEmbedLastOffset == null;
|
||||
}
|
||||
// @block-end re-embed-existing
|
||||
|
||||
// @block-start search-with-old-vector
|
||||
const oldVectorResults = await client.query(COLLECTION, {
|
||||
query: {
|
||||
text: "my query",
|
||||
model: OLD_MODEL,
|
||||
},
|
||||
using: OLD_VECTOR,
|
||||
limit: 10,
|
||||
});
|
||||
// @block-end search-with-old-vector
|
||||
|
||||
// @block-start search-with-new-vector
|
||||
const newVectorResults = await client.query(COLLECTION, {
|
||||
query: {
|
||||
text: "my query",
|
||||
model: NEW_MODEL,
|
||||
},
|
||||
using: NEW_VECTOR,
|
||||
limit: 10,
|
||||
});
|
||||
// @block-end search-with-new-vector
|
||||
|
||||
// @block-start delete-old-named-vector
|
||||
await client.deleteVectorName(COLLECTION, OLD_VECTOR);
|
||||
// @block-end delete-old-named-vector
|
||||
|
||||
Reference in New Issue
Block a user