mirror of
https://github.com/qdrant/landing_page.git
synced 2026-09-30 16:38:31 +02:00
Add TypeScript examples for Documentation & Quickstart (#376)
* Add TS examples for Documentation/Concepts * Minor FAQ improvements: spelling & formatting (#318) * Minor spelling and formatting improvements in FAQ * Add unused RAM is wasted RAM quote, improve memory text * Add TypeScript version of Quickstart (#378) * Minor FAQ improvements: spelling & formatting (#318) * Minor spelling and formatting improvements in FAQ * Add unused RAM is wasted RAM quote, improve memory text * Add TS version of quickstart --------- Co-authored-by: Tim Visée <tim@visee.me> * Add TS examples to Documentation / Guides * Fix terminology * Add TS examples to Documentation / Cloud * Add TS version of bulk upload tutorial * Format tutorials with black * Move author data to [params.author] (#385) * Add TS examples for Documentation/Concepts * Add TypeScript version of Quickstart (#378) * Minor FAQ improvements: spelling & formatting (#318) * Minor spelling and formatting improvements in FAQ * Add unused RAM is wasted RAM quote, improve memory text * Add TS version of quickstart --------- Co-authored-by: Tim Visée <tim@visee.me> * Add TS examples to Documentation / Guides * Fix terminology * Add TS examples to Documentation / Cloud * Add TS version of bulk upload tutorial * Format tutorials with black --------- Co-authored-by: Tim Visée <tim@visee.me>
This commit is contained in:
co-authored by
Tim Visée
parent
c79061a2ef
commit
a389d78bd2
@@ -61,7 +61,7 @@ from aleph_alpha_client import (
|
||||
AsyncClient,
|
||||
SemanticEmbeddingRequest,
|
||||
SemanticRepresentation,
|
||||
ImagePrompt
|
||||
ImagePrompt,
|
||||
)
|
||||
|
||||
from glob import glob
|
||||
@@ -69,7 +69,7 @@ from glob import glob
|
||||
ids, vectors, payloads = [], [], []
|
||||
async with AsyncClient(token=aa_token) as client:
|
||||
for i, image_path in enumerate(glob("./val2017/*.jpg")):
|
||||
# Convert the JPEG file into the embedding by calling
|
||||
# Convert the JPEG file into the embedding by calling
|
||||
# Aleph Alpha API
|
||||
prompt = ImagePrompt.from_file(image_path)
|
||||
prompt = Prompt.from_image(prompt)
|
||||
@@ -79,9 +79,7 @@ async with AsyncClient(token=aa_token) as client:
|
||||
"compress_to_size": 128,
|
||||
}
|
||||
query_request = SemanticEmbeddingRequest(**query_params)
|
||||
query_response = await client.semantic_embed(
|
||||
request=query_request, model=model
|
||||
)
|
||||
query_response = await client.semantic_embed(request=query_request, model=model)
|
||||
|
||||
# Finally store the id, vector and the payload
|
||||
ids.append(i)
|
||||
@@ -99,7 +97,7 @@ from qdrant_client.http.models import Batch, VectorParams, Distance
|
||||
|
||||
qdrant_client = qdrant_client.Qdrant.Client()
|
||||
qdrant_client.recreate_collection(
|
||||
collection_name="COCO"
|
||||
collection_name="COCO",
|
||||
vector_params=VectorParams(
|
||||
size=len(vectors[0]),
|
||||
distance=Distance.COSINE,
|
||||
@@ -135,9 +133,7 @@ async with AsyncCliet(token=aa_token) as client:
|
||||
"compress_to_size": 128,
|
||||
}
|
||||
query_request = SemanticEmbeddingRequest(**query_params)
|
||||
query_response = await client.semantic_embed(
|
||||
request=query_request, model=model
|
||||
)
|
||||
query_response = await client.semantic_embed(request=query_request, model=model)
|
||||
|
||||
results = qdrant.search(
|
||||
collection_name="COCO",
|
||||
@@ -155,7 +151,7 @@ Here are the results:
|
||||
and Spanish. Your search is not only multimodal, but also multilingual, without any need for translations.
|
||||
|
||||
```python
|
||||
text= "Surfing"
|
||||
text = "Surfing"
|
||||
|
||||
async with AsyncClient(token=aa_token) as client:
|
||||
query_params = {
|
||||
@@ -164,9 +160,7 @@ async with AsyncClient(token=aa_token) as client:
|
||||
"compres_to_size": 128,
|
||||
}
|
||||
query_request = SemanticEmbeddingRequest(**query_params)
|
||||
query_response = await client.semantic_embed(
|
||||
request=query_request, model=model
|
||||
)
|
||||
query_response = await client.semantic_embed(request=query_request, model=model)
|
||||
|
||||
results = qdrant.search(
|
||||
collection_name="COCO",
|
||||
|
||||
@@ -41,15 +41,16 @@ from qdrant_client import models
|
||||
import qdrant_client
|
||||
import asyncio
|
||||
|
||||
|
||||
async def main():
|
||||
client = qdrant_client.AsyncQdrantClient("localhost")
|
||||
|
||||
|
||||
# Create a collection
|
||||
await client.create_collection(
|
||||
collection_name="my_collection",
|
||||
vectors_config=models.VectorParams(size=4, distance=models.Distance.COSINE),
|
||||
)
|
||||
|
||||
|
||||
# Insert a vector
|
||||
await client.upsert(
|
||||
collection_name="my_collection",
|
||||
@@ -61,19 +62,20 @@ async def main():
|
||||
},
|
||||
vector=[0.9, 0.1, 0.1, 0.5],
|
||||
),
|
||||
]
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
# Search for nearest neighbors
|
||||
points = await client.search(
|
||||
collection_name="my_collection",
|
||||
query_vector=[0.9, 0.1, 0.1, 0.5],
|
||||
limit=2,
|
||||
)
|
||||
|
||||
|
||||
# Your async code using AsyncQdrantClient might be put here
|
||||
# ...
|
||||
|
||||
|
||||
asyncio.run(main())
|
||||
```
|
||||
|
||||
|
||||
@@ -40,7 +40,7 @@ from qdrant_client import QdrantClient, models
|
||||
|
||||
client = QdrantClient("localhost", port=6333)
|
||||
|
||||
client.recreate_collection(
|
||||
client.create_collection(
|
||||
collection_name="{collection_name}",
|
||||
vectors_config=models.VectorParams(size=768, distance=models.Distance.COSINE),
|
||||
optimizers_config=models.OptimizersConfigDiff(
|
||||
@@ -49,6 +49,22 @@ client.recreate_collection(
|
||||
)
|
||||
```
|
||||
|
||||
```typescript
|
||||
import { QdrantClient } from "@qdrant/js-client-rest";
|
||||
|
||||
const client = new QdrantClient({ host: "localhost", port: 6333 });
|
||||
|
||||
client.createCollection("{collection_name}", {
|
||||
vectors: {
|
||||
size: 768,
|
||||
distance: "Cosine",
|
||||
},
|
||||
optimizers_config: {
|
||||
indexing_threshold: 0,
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
After upload is done, you can enable indexing by setting `indexing_threshold` to a desired value (default is 20000):
|
||||
|
||||
```http
|
||||
@@ -68,12 +84,22 @@ client = QdrantClient("localhost", port=6333)
|
||||
|
||||
client.update_collection(
|
||||
collection_name="{collection_name}",
|
||||
optimizer_config=models.OptimizersConfigDiff(
|
||||
indexing_threshold=20000
|
||||
)
|
||||
optimizer_config=models.OptimizersConfigDiff(indexing_threshold=20000),
|
||||
)
|
||||
```
|
||||
|
||||
```typescript
|
||||
import { QdrantClient } from "@qdrant/js-client-rest";
|
||||
|
||||
const client = new QdrantClient({ host: "localhost", port: 6333 });
|
||||
|
||||
client.updateCollection("{collection_name}", {
|
||||
optimizers_config: {
|
||||
indexing_threshold: 20000,
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
## Upload directly to disk
|
||||
|
||||
When the vectors you upload do not all fit in RAM, you likely want to use
|
||||
@@ -117,9 +143,23 @@ from qdrant_client import QdrantClient, models
|
||||
|
||||
client = QdrantClient("localhost", port=6333)
|
||||
|
||||
client.recreate_collection(
|
||||
client.create_collection(
|
||||
collection_name="{collection_name}",
|
||||
vectors_config=models.VectorParams(size=768, distance=models.Distance.COSINE),
|
||||
shard_number=2,
|
||||
)
|
||||
```
|
||||
|
||||
```typescript
|
||||
import { QdrantClient } from "@qdrant/js-client-rest";
|
||||
|
||||
const client = new QdrantClient({ host: "localhost", port: 6333 });
|
||||
|
||||
client.createCollection("{collection_name}", {
|
||||
vectors: {
|
||||
size: 768,
|
||||
distance: "Cosine",
|
||||
},
|
||||
shard_number: 2,
|
||||
});
|
||||
```
|
||||
@@ -92,12 +92,10 @@ client.recreate_collection(
|
||||
collection_name=collection_name,
|
||||
vectors_config={
|
||||
"image": models.VectorParams(
|
||||
size=image_embeddings.shape[1],
|
||||
distance=models.Distance.COSINE
|
||||
size=image_embeddings.shape[1], distance=models.Distance.COSINE
|
||||
),
|
||||
"text": models.VectorParams(
|
||||
size=text_embeddings.shape[1],
|
||||
distance=models.Distance.COSINE
|
||||
size=text_embeddings.shape[1], distance=models.Distance.COSINE
|
||||
),
|
||||
},
|
||||
)
|
||||
@@ -136,7 +134,6 @@ As a response, we should see a similar output:
|
||||
|
||||
```python
|
||||
name='LAION-5B-1217055918586176-2023-07-04-11-51-24.snapshot' creation_time='2023-07-04T11:51:25' size=74202112
|
||||
|
||||
```
|
||||
|
||||
## List all snapshots
|
||||
@@ -150,7 +147,13 @@ print(snapshots)
|
||||
This endpoint exposes all the snapshots in the same format as before:
|
||||
|
||||
```python
|
||||
[SnapshotDescription(name='LAION-5B-1217055918586176-2023-07-04-11-51-24.snapshot', creation_time='2023-07-04T11:51:25', size=74202112)]
|
||||
[
|
||||
SnapshotDescription(
|
||||
name="LAION-5B-1217055918586176-2023-07-04-11-51-24.snapshot",
|
||||
creation_time="2023-07-04T11:51:25",
|
||||
size=74202112,
|
||||
)
|
||||
]
|
||||
```
|
||||
|
||||
We can use the same naming convention to create the URL to download it.
|
||||
|
||||
@@ -11,7 +11,7 @@ weight: 2
|
||||
This tutorial shows you how to build and deploy your own neural search service to look through descriptions of companies from [startups-list.com](https://www.startups-list.com/) and pick the most similar ones to your query.
|
||||
The website contains the company names, descriptions, locations, and a picture for each entry.
|
||||
|
||||
Alternatvely, you cna use such datasources as [Crunchbase](https://www.crunchbase.com/), but that would require obtaining an API key from them.
|
||||
Alternatively, you can use datasources such as [Crunchbase](https://www.crunchbase.com/), but that would require obtaining an API key from them.
|
||||
|
||||
Our neural search service will use [Fastembed](https://github.com/qdrant/fastembed) package to generate embeddings of text descriptions and [FastAPI](https://fastapi.tiangolo.com/) to serve the search API.
|
||||
Fastembed natively integrates with Qdrant client, so you can easily upload the data into Qdrant and perform search queries.
|
||||
@@ -106,7 +106,7 @@ Now you need to write a script to upload all startup data and vectors into the s
|
||||
# Import client library
|
||||
from qdrant_client import QdrantClient
|
||||
|
||||
qdrant_client = QdrantClient('http://localhost:6333')
|
||||
qdrant_client = QdrantClient("http://localhost:6333")
|
||||
```
|
||||
|
||||
3. Select model to encode your data.
|
||||
@@ -122,7 +122,7 @@ qdrant_client.set_model("sentence-transformers/all-MiniLM-L6-v2")
|
||||
|
||||
```python
|
||||
qdrant_client.recreate_collection(
|
||||
collection_name='startups',
|
||||
collection_name="startups",
|
||||
vectors_config=qdrant_client.get_fastembed_vector_params(),
|
||||
)
|
||||
```
|
||||
@@ -137,14 +137,14 @@ Additionally, you can specify extended configuration for our vectors, like `quan
|
||||
5. Read data from the file.
|
||||
|
||||
```python
|
||||
payload_path = os.path.join(DATA_DIR, 'startups_demo.json')
|
||||
payload_path = os.path.join(DATA_DIR, "startups_demo.json")
|
||||
metadata = []
|
||||
documents = []
|
||||
|
||||
with open(payload_path) as fd:
|
||||
for line in fd:
|
||||
obj = json.loads(line)
|
||||
documents.append(obj.pop('description'))
|
||||
documents.append(obj.pop("description"))
|
||||
metadata.append(obj)
|
||||
```
|
||||
|
||||
@@ -157,10 +157,10 @@ We will use `documents` to encode the data into vectors.
|
||||
|
||||
```python
|
||||
client.add(
|
||||
collection_name='startups',
|
||||
collection_name="startups",
|
||||
documents=documents,
|
||||
metadata=metadata,
|
||||
parallel=0, # Use all available CPU cores to encode data
|
||||
parallel=0, # Use all available CPU cores to encode data
|
||||
)
|
||||
```
|
||||
|
||||
@@ -178,10 +178,10 @@ You can monitor the progress of the encoding by passing tqdm progress bar to the
|
||||
from tqdm import tqdm
|
||||
|
||||
client.add(
|
||||
collection_name='startups',
|
||||
collection_name="startups",
|
||||
documents=documents,
|
||||
metadata=metadata,
|
||||
ids=tqdm(range(len(documents)))
|
||||
ids=tqdm(range(len(documents))),
|
||||
)
|
||||
```
|
||||
|
||||
@@ -201,19 +201,19 @@ Fastembed integration into qdrant client combines encoding and uploading into a
|
||||
```python
|
||||
from qdrant_client import QdrantClient
|
||||
|
||||
class NeuralSearcher:
|
||||
|
||||
class NeuralSearcher:
|
||||
def __init__(self, collection_name):
|
||||
self.collection_name = collection_name
|
||||
# initialize Qdrant client
|
||||
self.qdrant_client = QdrantClient('http://localhost:6333')
|
||||
self.qdrant_client.set_model('sentence-transformers/all-MiniLM-L6-v2')
|
||||
self.qdrant_client = QdrantClient("http://localhost:6333")
|
||||
self.qdrant_client.set_model("sentence-transformers/all-MiniLM-L6-v2")
|
||||
```
|
||||
|
||||
2. Write the search function.
|
||||
|
||||
```python
|
||||
def search(self, text: str):
|
||||
def search(self, text: str):
|
||||
search_result = self.qdrant_client.query(
|
||||
collection_name=self.collection_name,
|
||||
query_text=text,
|
||||
@@ -255,7 +255,6 @@ from qdrant_client.models import Filter
|
||||
limit=5
|
||||
)
|
||||
...
|
||||
|
||||
```
|
||||
|
||||
You have now created a class for neural search queries. Now wrap it up into a service.
|
||||
@@ -279,7 +278,7 @@ Create a file named `service.py` and specify the following.
|
||||
The service will have only one API endpoint and will look like this:
|
||||
|
||||
```python
|
||||
from fastapi import FastAPI
|
||||
from fastapi import FastAPI
|
||||
|
||||
# The file where NeuralSearcher is stored
|
||||
from neural_searcher import NeuralSearcher
|
||||
@@ -299,7 +298,6 @@ def search_startup(q: str):
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
uvicorn.run(app, host="0.0.0.0", port=8000)
|
||||
|
||||
```
|
||||
|
||||
3. Run the service.
|
||||
|
||||
@@ -67,20 +67,22 @@ This is a performance-optimized sentence embedding model and you can read more a
|
||||
4. Download and create a pre-trained sentence encoder.
|
||||
|
||||
```python
|
||||
model = SentenceTransformer('all-MiniLM-L6-v2', device="cuda") # or device="cpu" if you don't have a GPU
|
||||
model = SentenceTransformer(
|
||||
"all-MiniLM-L6-v2", device="cuda"
|
||||
) # or device="cpu" if you don't have a GPU
|
||||
```
|
||||
5. Read the raw data file.
|
||||
|
||||
```python
|
||||
df = pd.read_json('./startups_demo.json', lines=True)
|
||||
df = pd.read_json("./startups_demo.json", lines=True)
|
||||
```
|
||||
6. Encode all startup descriptions to create an embedding vector for each. Internally, the `encode` function will split the input into batches, which will significantly speed up the process.
|
||||
|
||||
```python
|
||||
vectors = model.encode([
|
||||
row.alt + ". " + row.description
|
||||
for row in df.itertuples()
|
||||
], show_progress_bar=True)
|
||||
vectors = model.encode(
|
||||
[row.alt + ". " + row.description for row in df.itertuples()],
|
||||
show_progress_bar=True,
|
||||
)
|
||||
```
|
||||
All of the descriptions are now converted into vectors. There are 40474 vectors of 384 dimensions. The output layer of the model has this dimension
|
||||
|
||||
@@ -92,7 +94,7 @@ vectors.shape
|
||||
7. Download the saved vectors into a new file named `startup_vectors.npy`
|
||||
|
||||
```python
|
||||
np.save('startup_vectors.npy', vectors, allow_pickle=False)
|
||||
np.save("startup_vectors.npy", vectors, allow_pickle=False)
|
||||
```
|
||||
|
||||
## Run Qdrant in Docker
|
||||
@@ -144,14 +146,14 @@ Now you need to write a script to upload all startup data and vectors into the s
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.models import VectorParams, Distance
|
||||
|
||||
qdrant_client = QdrantClient('http://localhost:6333')
|
||||
qdrant_client = QdrantClient("http://localhost:6333")
|
||||
```
|
||||
|
||||
3. Related vectors need to be added to a collection. Create a new collection for your startup vectors.
|
||||
|
||||
```python
|
||||
qdrant_client.recreate_collection(
|
||||
collection_name='startups',
|
||||
collection_name="startups",
|
||||
vectors_config=VectorParams(size=384, distance=Distance.COSINE),
|
||||
)
|
||||
```
|
||||
@@ -171,25 +173,25 @@ The Qdrant client library defines a special function that allows you to load dat
|
||||
However, since there may be too much data to fit a single computer memory, the function takes an iterator over the data as input.
|
||||
|
||||
```python
|
||||
fd = open('./startups_demo.json')
|
||||
fd = open("./startups_demo.json")
|
||||
|
||||
# payload is now an iterator over startup data
|
||||
payload = map(json.loads, fd)
|
||||
|
||||
# Load all vectors into memory, numpy array works as iterable for itself.
|
||||
# Other option would be to use Mmap, if you don't want to load all data into RAM
|
||||
vectors = np.load('./startup_vectors.npy')
|
||||
vectors = np.load("./startup_vectors.npy")
|
||||
```
|
||||
|
||||
5. Upload the data
|
||||
|
||||
```python
|
||||
qdrant_client.upload_collection(
|
||||
collection_name='startups',
|
||||
collection_name="startups",
|
||||
vectors=vectors,
|
||||
payload=payload,
|
||||
ids=None, # Vector ids will be assigned automatically
|
||||
batch_size=256 # How many vectors will be uploaded in a single request?
|
||||
batch_size=256, # How many vectors will be uploaded in a single request?
|
||||
)
|
||||
```
|
||||
|
||||
@@ -209,19 +211,18 @@ from sentence_transformers import SentenceTransformer
|
||||
|
||||
|
||||
class NeuralSearcher:
|
||||
|
||||
def __init__(self, collection_name):
|
||||
self.collection_name = collection_name
|
||||
# Initialize encoder model
|
||||
self.model = SentenceTransformer('all-MiniLM-L6-v2', device='cpu')
|
||||
self.model = SentenceTransformer("all-MiniLM-L6-v2", device="cpu")
|
||||
# initialize Qdrant client
|
||||
self.qdrant_client = QdrantClient('http://localhost:6333')
|
||||
self.qdrant_client = QdrantClient("http://localhost:6333")
|
||||
```
|
||||
|
||||
2. Write the search function.
|
||||
|
||||
```python
|
||||
def search(self, text: str):
|
||||
def search(self, text: str):
|
||||
# Convert text query into vector
|
||||
vector = self.model.encode(text).tolist()
|
||||
|
||||
@@ -267,7 +268,6 @@ from qdrant_client.models import Filter
|
||||
limit=5
|
||||
)
|
||||
...
|
||||
|
||||
```
|
||||
|
||||
You have now created a class for neural search queries. Now wrap it up into a service.
|
||||
@@ -291,7 +291,7 @@ Create a file named `service.py` and specify the following.
|
||||
The service will have only one API endpoint and will look like this:
|
||||
|
||||
```python
|
||||
from fastapi import FastAPI
|
||||
from fastapi import FastAPI
|
||||
|
||||
# The file where NeuralSearcher is stored
|
||||
from neural_searcher import NeuralSearcher
|
||||
@@ -311,7 +311,6 @@ def search_startup(q: str):
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
uvicorn.run(app, host="0.0.0.0", port=8000)
|
||||
|
||||
```
|
||||
|
||||
3. Run the service.
|
||||
|
||||
@@ -42,7 +42,7 @@ from sentence_transformers import SentenceTransformer
|
||||
The [Sentence Transformers](https://www.sbert.net/index.html) framework contains many embedding models. However, [all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) is the fastest encoder for this tutorial.
|
||||
|
||||
```python
|
||||
encoder = SentenceTransformer('all-MiniLM-L6-v2')
|
||||
encoder = SentenceTransformer("all-MiniLM-L6-v2")
|
||||
```
|
||||
|
||||
## 2. Add the dataset
|
||||
@@ -51,19 +51,84 @@ encoder = SentenceTransformer('all-MiniLM-L6-v2')
|
||||
|
||||
```python
|
||||
documents = [
|
||||
{ "name": "The Time Machine", "description": "A man travels through time and witnesses the evolution of humanity.", "author": "H.G. Wells", "year": 1895 },
|
||||
{ "name": "Ender's Game", "description": "A young boy is trained to become a military leader in a war against an alien race.", "author": "Orson Scott Card", "year": 1985 },
|
||||
{ "name": "Brave New World", "description": "A dystopian society where people are genetically engineered and conditioned to conform to a strict social hierarchy.", "author": "Aldous Huxley", "year": 1932 },
|
||||
{ "name": "The Hitchhiker's Guide to the Galaxy", "description": "A comedic science fiction series following the misadventures of an unwitting human and his alien friend.", "author": "Douglas Adams", "year": 1979 },
|
||||
{ "name": "Dune", "description": "A desert planet is the site of political intrigue and power struggles.", "author": "Frank Herbert", "year": 1965 },
|
||||
{ "name": "Foundation", "description": "A mathematician develops a science to predict the future of humanity and works to save civilization from collapse.", "author": "Isaac Asimov", "year": 1951 },
|
||||
{ "name": "Snow Crash", "description": "A futuristic world where the internet has evolved into a virtual reality metaverse.", "author": "Neal Stephenson", "year": 1992 },
|
||||
{ "name": "Neuromancer", "description": "A hacker is hired to pull off a near-impossible hack and gets pulled into a web of intrigue.", "author": "William Gibson", "year": 1984 },
|
||||
{ "name": "The War of the Worlds", "description": "A Martian invasion of Earth throws humanity into chaos.", "author": "H.G. Wells", "year": 1898 },
|
||||
{ "name": "The Hunger Games", "description": "A dystopian society where teenagers are forced to fight to the death in a televised spectacle.", "author": "Suzanne Collins", "year": 2008 },
|
||||
{ "name": "The Andromeda Strain", "description": "A deadly virus from outer space threatens to wipe out humanity.", "author": "Michael Crichton", "year": 1969 },
|
||||
{ "name": "The Left Hand of Darkness", "description": "A human ambassador is sent to a planet where the inhabitants are genderless and can change gender at will.", "author": "Ursula K. Le Guin", "year": 1969 },
|
||||
{ "name": "The Three-Body Problem", "description": "Humans encounter an alien civilization that lives in a dying system.", "author": "Liu Cixin", "year": 2008 }
|
||||
{
|
||||
"name": "The Time Machine",
|
||||
"description": "A man travels through time and witnesses the evolution of humanity.",
|
||||
"author": "H.G. Wells",
|
||||
"year": 1895,
|
||||
},
|
||||
{
|
||||
"name": "Ender's Game",
|
||||
"description": "A young boy is trained to become a military leader in a war against an alien race.",
|
||||
"author": "Orson Scott Card",
|
||||
"year": 1985,
|
||||
},
|
||||
{
|
||||
"name": "Brave New World",
|
||||
"description": "A dystopian society where people are genetically engineered and conditioned to conform to a strict social hierarchy.",
|
||||
"author": "Aldous Huxley",
|
||||
"year": 1932,
|
||||
},
|
||||
{
|
||||
"name": "The Hitchhiker's Guide to the Galaxy",
|
||||
"description": "A comedic science fiction series following the misadventures of an unwitting human and his alien friend.",
|
||||
"author": "Douglas Adams",
|
||||
"year": 1979,
|
||||
},
|
||||
{
|
||||
"name": "Dune",
|
||||
"description": "A desert planet is the site of political intrigue and power struggles.",
|
||||
"author": "Frank Herbert",
|
||||
"year": 1965,
|
||||
},
|
||||
{
|
||||
"name": "Foundation",
|
||||
"description": "A mathematician develops a science to predict the future of humanity and works to save civilization from collapse.",
|
||||
"author": "Isaac Asimov",
|
||||
"year": 1951,
|
||||
},
|
||||
{
|
||||
"name": "Snow Crash",
|
||||
"description": "A futuristic world where the internet has evolved into a virtual reality metaverse.",
|
||||
"author": "Neal Stephenson",
|
||||
"year": 1992,
|
||||
},
|
||||
{
|
||||
"name": "Neuromancer",
|
||||
"description": "A hacker is hired to pull off a near-impossible hack and gets pulled into a web of intrigue.",
|
||||
"author": "William Gibson",
|
||||
"year": 1984,
|
||||
},
|
||||
{
|
||||
"name": "The War of the Worlds",
|
||||
"description": "A Martian invasion of Earth throws humanity into chaos.",
|
||||
"author": "H.G. Wells",
|
||||
"year": 1898,
|
||||
},
|
||||
{
|
||||
"name": "The Hunger Games",
|
||||
"description": "A dystopian society where teenagers are forced to fight to the death in a televised spectacle.",
|
||||
"author": "Suzanne Collins",
|
||||
"year": 2008,
|
||||
},
|
||||
{
|
||||
"name": "The Andromeda Strain",
|
||||
"description": "A deadly virus from outer space threatens to wipe out humanity.",
|
||||
"author": "Michael Crichton",
|
||||
"year": 1969,
|
||||
},
|
||||
{
|
||||
"name": "The Left Hand of Darkness",
|
||||
"description": "A human ambassador is sent to a planet where the inhabitants are genderless and can change gender at will.",
|
||||
"author": "Ursula K. Le Guin",
|
||||
"year": 1969,
|
||||
},
|
||||
{
|
||||
"name": "The Three-Body Problem",
|
||||
"description": "Humans encounter an alien civilization that lives in a dying system.",
|
||||
"author": "Liu Cixin",
|
||||
"year": 2008,
|
||||
},
|
||||
]
|
||||
```
|
||||
|
||||
@@ -72,7 +137,7 @@ documents = [
|
||||
You need to tell Qdrant where to store embeddings. This is a basic demo, so your local computer will use its memory as temporary storage.
|
||||
|
||||
```python
|
||||
qdrant = QdrantClient(":memory:")
|
||||
qdrant = QdrantClient(":memory:")
|
||||
```
|
||||
|
||||
## 4. Create a collection
|
||||
@@ -81,11 +146,11 @@ All data in Qdrant is organized by collections. In this case, you are storing bo
|
||||
|
||||
```python
|
||||
qdrant.recreate_collection(
|
||||
collection_name="my_books",
|
||||
vectors_config=models.VectorParams(
|
||||
size=encoder.get_sentence_embedding_dimension(), # Vector size is defined by used model
|
||||
distance=models.Distance.COSINE
|
||||
)
|
||||
collection_name="my_books",
|
||||
vectors_config=models.VectorParams(
|
||||
size=encoder.get_sentence_embedding_dimension(), # Vector size is defined by used model
|
||||
distance=models.Distance.COSINE,
|
||||
),
|
||||
)
|
||||
```
|
||||
|
||||
@@ -102,14 +167,13 @@ Tell the database to upload `documents` to the `my_books` collection. This will
|
||||
|
||||
```python
|
||||
qdrant.upload_records(
|
||||
collection_name="my_books",
|
||||
records=[
|
||||
models.Record(
|
||||
id=idx,
|
||||
vector=encoder.encode(doc["description"]).tolist(),
|
||||
payload=doc
|
||||
) for idx, doc in enumerate(documents)
|
||||
]
|
||||
collection_name="my_books",
|
||||
records=[
|
||||
models.Record(
|
||||
id=idx, vector=encoder.encode(doc["description"]).tolist(), payload=doc
|
||||
)
|
||||
for idx, doc in enumerate(documents)
|
||||
],
|
||||
)
|
||||
```
|
||||
|
||||
@@ -119,12 +183,12 @@ Now that the data is stored in Qdrant, you can ask it questions and receive sema
|
||||
|
||||
```python
|
||||
hits = qdrant.search(
|
||||
collection_name="my_books",
|
||||
query_vector=encoder.encode("alien invasion").tolist(),
|
||||
limit=3
|
||||
collection_name="my_books",
|
||||
query_vector=encoder.encode("alien invasion").tolist(),
|
||||
limit=3,
|
||||
)
|
||||
for hit in hits:
|
||||
print(hit.payload, "score:", hit.score)
|
||||
print(hit.payload, "score:", hit.score)
|
||||
```
|
||||
|
||||
**Response:**
|
||||
@@ -143,22 +207,15 @@ How about the most recent book from the early 2000s?
|
||||
|
||||
```python
|
||||
hits = qdrant.search(
|
||||
collection_name="my_books",
|
||||
query_vector=encoder.encode("alien invasion").tolist(),
|
||||
query_filter=models.Filter(
|
||||
must=[
|
||||
models.FieldCondition(
|
||||
key="year",
|
||||
range=models.Range(
|
||||
gte=2000
|
||||
)
|
||||
)
|
||||
]
|
||||
),
|
||||
limit=1
|
||||
collection_name="my_books",
|
||||
query_vector=encoder.encode("alien invasion").tolist(),
|
||||
query_filter=models.Filter(
|
||||
must=[models.FieldCondition(key="year", range=models.Range(gte=2000))]
|
||||
),
|
||||
limit=1,
|
||||
)
|
||||
for hit in hits:
|
||||
print(hit.payload, "score:", hit.score)
|
||||
print(hit.payload, "score:", hit.score)
|
||||
```
|
||||
|
||||
**Response:**
|
||||
|
||||
Reference in New Issue
Block a user