mirror of
https://github.com/qdrant/landing_page.git
synced 2026-10-09 12:58:31 +02:00
258 lines
8.2 KiB
Python
258 lines
8.2 KiB
Python
# @block-start client-connection
|
||
from qdrant_client import QdrantClient, models
|
||
|
||
# Replace url and api_key with your own from https://cloud.qdrant.io
|
||
client = QdrantClient(
|
||
url="https://xyz-example.qdrant.io:6333",
|
||
api_key="<your-api-key>",
|
||
cloud_inference=True
|
||
)
|
||
# @block-end client-connection
|
||
|
||
# @hide-start
|
||
# data and text normalization are not the lesson of this tutorial:
|
||
# the full CHUNKS list and normalize() live in the tutorial notebook
|
||
CHUNKS = [
|
||
{
|
||
"url": "https://qdrant.tech/documentation/tutorials-operations/secure-qdrant/",
|
||
"anchor": "prerequisites",
|
||
"chunk_num": 0,
|
||
"text": "Prerequisites - Docker and Docker Compose installed - `curl` available in your terminal ...",
|
||
},
|
||
{
|
||
"url": "https://qdrant.tech/documentation/tutorials-operations/secure-qdrant/",
|
||
"anchor": "step-3-enable-an-admin-api-key",
|
||
"chunk_num": 0,
|
||
"text": "Step 3: Enable an Admin API Key Without enabling authentication, anyone with network access ...",
|
||
},
|
||
]
|
||
|
||
import re
|
||
import unicodedata
|
||
|
||
def normalize(text):
|
||
text = unicodedata.normalize("NFKC", text)
|
||
text = text.translate(dict.fromkeys(map(ord, "")))
|
||
return re.sub(r"\s+", " ", text).strip()
|
||
# @hide-end
|
||
|
||
# @block-start create-collection
|
||
MODEL = "sentence-transformers/all-MiniLM-L6-v2"
|
||
PIPELINE = "docs-prep-pipeline-v1"
|
||
COLLECTION = "docs-sync-tutorial"
|
||
|
||
client.create_collection(
|
||
COLLECTION,
|
||
vectors_config=models.VectorParams(
|
||
size=384, # all-MiniLM-L6-v2 output dimension
|
||
distance=models.Distance.COSINE,
|
||
),
|
||
metadata={"embedding_model": MODEL, "pipeline_version": PIPELINE},
|
||
)
|
||
# @block-end create-collection
|
||
|
||
# @block-start check-gate
|
||
def check_gate():
|
||
# compare this pipeline's constants against what the collection records about itself
|
||
meta = client.get_collection(COLLECTION).config.metadata or {}
|
||
|
||
if meta.get("embedding_model") != MODEL or meta.get("pipeline_version") != PIPELINE:
|
||
raise RuntimeError(f"collection was built by {meta}: full re-embed into a fresh collection required")
|
||
# @block-end check-gate
|
||
|
||
# @block-start identity-and-fingerprint
|
||
import hashlib
|
||
import uuid
|
||
from datetime import datetime, timezone
|
||
|
||
def content_hash(text):
|
||
return hashlib.sha256(text.encode()).hexdigest()
|
||
|
||
def point_id(url, anchor, num):
|
||
# NAMESPACE_URL is a fixed constant uuid5 requires; it marks the input as a URL-like name
|
||
return str(uuid.uuid5(uuid.NAMESPACE_URL, f"{url}#{anchor}::{num}"))
|
||
|
||
def prepare_chunks_for_sync(chunks):
|
||
"""Derive both values (and the section address) for every raw chunk."""
|
||
out = []
|
||
for c in chunks:
|
||
text = normalize(c["text"])
|
||
out.append({
|
||
**c,
|
||
"text": text,
|
||
"section_url": f"{c['url']}#{c['anchor']}" if c["anchor"] else c["url"],
|
||
"content_hash": content_hash(text),
|
||
"point_id": point_id(c["url"], c["anchor"], c["chunk_num"]),
|
||
})
|
||
return out
|
||
# @block-end identity-and-fingerprint
|
||
|
||
# @block-start payload
|
||
def payload(chunk, last_updated=None):
|
||
return {
|
||
"url": chunk["url"],
|
||
"anchor": chunk["anchor"],
|
||
"chunk_num": chunk["chunk_num"],
|
||
"section_url": chunk["section_url"],
|
||
"text": chunk["text"],
|
||
"content_hash": chunk["content_hash"],
|
||
"last_updated": last_updated or datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
||
}
|
||
# @block-end payload
|
||
|
||
# @block-start payload-indexes
|
||
for field in ("content_hash", "url", "section_url"):
|
||
client.create_payload_index(COLLECTION, field, models.PayloadSchemaType.KEYWORD)
|
||
# @block-end payload-indexes
|
||
|
||
# @block-start populate
|
||
client.upsert(COLLECTION, points=[
|
||
models.PointStruct(
|
||
id=c["point_id"],
|
||
vector=models.Document(text=c["text"], model=MODEL),
|
||
payload=payload(c),
|
||
)
|
||
for c in prepare_chunks_for_sync(CHUNKS)
|
||
], wait=True)
|
||
# @block-end populate
|
||
|
||
# @block-start search
|
||
QUERY = "Where exactly to set `QDRANT__SERVICE__API_KEY` variable to enable authentication for a self-hosted Qdrant?"
|
||
|
||
client.query_points(
|
||
COLLECTION,
|
||
query=models.Document(text=QUERY, model=MODEL),
|
||
limit=3,
|
||
with_payload=["section_url", "text"],
|
||
)
|
||
# @block-end search
|
||
|
||
# @hide-start
|
||
# the simulated month of edits (LATEST_CHUNKS) is spelled out in the tutorial and the notebook
|
||
LATEST_CHUNKS = prepare_chunks_for_sync(CHUNKS)
|
||
# @hide-end
|
||
|
||
# @block-start split-by-state
|
||
def split_by_state(latest_chunks):
|
||
"""Compare the incoming chunk list to the collection: who is unchanged, changed, or unknown."""
|
||
incoming = {c["point_id"]: c for c in latest_chunks}
|
||
|
||
stored = {}
|
||
points = client.retrieve(
|
||
COLLECTION,
|
||
ids=list(incoming),
|
||
with_payload=["content_hash"],
|
||
with_vectors=False,
|
||
)
|
||
for p in points:
|
||
stored[str(p.id)] = p.payload["content_hash"]
|
||
|
||
unchanged, content_changed, unknown_ids = [], [], []
|
||
for pid, c in incoming.items():
|
||
if stored.get(pid) == c["content_hash"]:
|
||
unchanged.append(c)
|
||
elif pid in stored:
|
||
content_changed.append(c)
|
||
else:
|
||
unknown_ids.append(c)
|
||
|
||
return incoming, unchanged, content_changed, unknown_ids
|
||
|
||
incoming_ids, unchanged, content_changed, unknown_ids = split_by_state(LATEST_CHUNKS)
|
||
# @block-end split-by-state
|
||
|
||
# @block-start re-embed-changed
|
||
def re_embed_changed(content_changed):
|
||
if not content_changed:
|
||
return
|
||
client.upsert(COLLECTION,
|
||
points=[
|
||
models.PointStruct(
|
||
id=c["point_id"],
|
||
vector=models.Document(text=c["text"], model=MODEL),
|
||
payload=payload(c),
|
||
)
|
||
for c in content_changed],
|
||
wait=True)
|
||
# @block-end re-embed-changed
|
||
|
||
# @block-start reuse-or-add
|
||
def reuse_or_add(unknown_ids):
|
||
"""Reuse an existing embedding when the same text is already stored; embed only what is new."""
|
||
reused, added = 0, 0
|
||
|
||
for c in unknown_ids:
|
||
same_text = models.Filter(must=[
|
||
models.FieldCondition(
|
||
key="content_hash",
|
||
match=models.MatchValue(value=c["content_hash"]),
|
||
)
|
||
])
|
||
hits, _ = client.scroll(
|
||
COLLECTION,
|
||
scroll_filter=same_text,
|
||
limit=1,
|
||
with_payload=["last_updated"],
|
||
with_vectors=True,
|
||
)
|
||
|
||
if hits: # same text, new address: copy the vector, keep its last_updated
|
||
point = models.PointStruct(
|
||
id=c["point_id"],
|
||
vector=hits[0].vector,
|
||
payload=payload(c, hits[0].payload["last_updated"]),
|
||
)
|
||
reused += 1
|
||
else: # genuinely new content: embed and insert
|
||
point = models.PointStruct(
|
||
id=c["point_id"],
|
||
vector=models.Document(text=c["text"], model=MODEL),
|
||
payload=payload(c),
|
||
)
|
||
added += 1
|
||
|
||
client.upsert(COLLECTION, points=[point], wait=True)
|
||
|
||
return reused, added
|
||
# @block-end reuse-or-add
|
||
|
||
# @block-start delete-gone
|
||
def delete_gone(incoming_ids):
|
||
"""Remove every point the current crawl no longer contains. Returns how many."""
|
||
if not incoming_ids:
|
||
raise ValueError("Refusing to delete from an empty source snapshot.")
|
||
|
||
stale = models.Filter(must_not=[models.HasIdCondition(has_id=list(incoming_ids))])
|
||
|
||
to_delete = client.count(COLLECTION, count_filter=stale).count
|
||
|
||
# potential check against a threshold to avoid accidental mass deletion could be added here
|
||
client.delete(COLLECTION, points_selector=models.FilterSelector(filter=stale), wait=True)
|
||
return to_delete
|
||
# @block-end delete-gone
|
||
|
||
# @block-start sync
|
||
def sync(latest_chunks):
|
||
check_gate() # refuse to mix embedding models or pipeline versions
|
||
|
||
chunks = prepare_chunks_for_sync(latest_chunks)
|
||
incoming_ids, unchanged, content_changed, unknown_ids = split_by_state(chunks)
|
||
|
||
re_embed_changed(content_changed)
|
||
reused, added = reuse_or_add(unknown_ids)
|
||
deleted = delete_gone(incoming_ids)
|
||
|
||
return {
|
||
"unchanged": len(unchanged),
|
||
"re-embedded": len(content_changed),
|
||
"reused_embedding": reused,
|
||
"added": added,
|
||
"deleted": deleted,
|
||
}
|
||
# @block-end sync
|
||
|
||
# @block-start run-sync
|
||
run = sync(LATEST_CHUNKS)
|
||
print(run)
|
||
# @block-end run-sync
|