mirror of
https://github.com/qdrant/landing_page.git
synced 2026-09-25 22:18:30 +02:00
Subsection for cloud documentation (#98)
* changed docs sidebar and breadcrumbs so it would work with subsections as well * buttons to switch between qdrant and cloud documentation * changed a link for editting on github for cloud doc pages * docs auto-sync * bold text on an active button * update of sync script * docs auto-sync * temporary use other branch for sync * temporary use other branch for sync * temporary use other branch for sync * temporary use other branch for sync * docs auto-sync * temporary use other branch for sync * temporary use other branch for sync * temporary use other branch for sync * docs auto-sync * styles * docs auto-sync Co-authored-by: qdrant <qdrant@users.noreply.github.com> Co-authored-by: Andrey Vasnetsov <andrey@vasnetsov.com>
This commit is contained in:
trean
co-authored by
qdrant
Andrey Vasnetsov
qdrant
Andrey Vasnetsov
parent
9ca90b2789
commit
ef072cce34
@@ -16,9 +16,18 @@ jobs:
|
||||
- name: Run a one-line script
|
||||
run: |
|
||||
bash -x automation/doc-sync.sh
|
||||
if [[ `git status --porcelain` ]]; then
|
||||
# Changes
|
||||
echo "Changes detected"
|
||||
else
|
||||
# No changes
|
||||
echo "No changes detected"
|
||||
exit 0
|
||||
fi
|
||||
git config --global user.name 'qdrant'
|
||||
git config --global user.email 'qdrant@users.noreply.github.com'
|
||||
git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY
|
||||
git checkout $GITHUB_HEAD_REF
|
||||
git add qdrant-landing/content/documentation/*.md
|
||||
git add qdrant-landing/content/documentation/cloud/*.md
|
||||
git commit -am "docs auto-sync" && git push --force || true
|
||||
|
||||
+22
-12
@@ -5,23 +5,33 @@ set -e
|
||||
DOC_REP=docs
|
||||
git clone https://github.com/qdrant/$DOC_REP.git
|
||||
|
||||
|
||||
# ToDo: remove before merge
|
||||
echo $(cd $DOC_REP && git pull && git checkout cloud-docs)
|
||||
|
||||
if [ -d $DOC_REP ]; then
|
||||
DOC_VERSION=$(grep -o 'docVersion = .*' ./qdrant-landing/config.toml | awk -F'"' '/(docVersion = )"([^"]+)"/{ print $2 }')
|
||||
QDRANT_DOC_VERSION=$(grep -o 'docVersion = .*' ./qdrant-landing/config.toml | awk -F'"' '/(docVersion = )"([^"]+)"/{ print $2 }')
|
||||
echo "Qdrant doc version: $QDRANT_DOC_VERSION"
|
||||
CLOUD_DOC_VERSION=$(grep -o 'cloudDocVersion = .*' ./qdrant-landing/config.toml | awk -F'"' '/(cloudDocVersion = )"([^"]+)"/{ print $2 }')
|
||||
echo "Cloud doc version: $CLOUD_DOC_VERSION"
|
||||
fi;
|
||||
|
||||
DOC_DESTINATION=./qdrant-landing/content/documentation
|
||||
DOC_SOURCE=./docs/qdrant/$DOC_VERSION
|
||||
QDRANT_DOC_DESTINATION=./qdrant-landing/content/documentation
|
||||
QDRANT_DOC_SOURCE=./docs/qdrant/$QDRANT_DOC_VERSION
|
||||
|
||||
DIFFER=$(diff -qr qdrant-landing/content/documentation docs/qdrant/"$DOC_VERSION" | cat);
|
||||
CLOUD_DOC_DESTINATION=$QDRANT_DOC_DESTINATION/cloud
|
||||
CLOUD_DOC_SOURCE=./docs/cloud/$CLOUD_DOC_VERSION
|
||||
|
||||
#if there is no changes, script just exits with code 0
|
||||
if [ -z "$DIFFER" ]; then
|
||||
echo "Sync is not needed, files are identical"
|
||||
rm -rf ./$DOC_REP
|
||||
exit 0;
|
||||
if [ ! -d "$CLOUD_DOC_DESTINATION" ]; then
|
||||
mkdir -p "$CLOUD_DOC_DESTINATION"
|
||||
fi;
|
||||
|
||||
#updates docs
|
||||
rm -rf $DOC_DESTINATION
|
||||
mv "$DOC_SOURCE" "$DOC_DESTINATION"
|
||||
rm -rf ./$DOC_REP
|
||||
rm -rf $QDRANT_DOC_DESTINATION
|
||||
mv "$QDRANT_DOC_SOURCE" "$QDRANT_DOC_DESTINATION"
|
||||
|
||||
#updates cloud docs
|
||||
rm -rf $CLOUD_DOC_DESTINATION
|
||||
mv "$CLOUD_DOC_SOURCE" "$CLOUD_DOC_DESTINATION"
|
||||
|
||||
rm -rf ./$DOC_REP
|
||||
|
||||
@@ -51,8 +51,10 @@ keywords = "search engine, vector database, neural network, matching, filter, Sa
|
||||
gdpr = "We use cookies to learn more about you. At any time you can delete or block cookies through your browser settings."
|
||||
|
||||
githubDocPrefix = "https://github.com/qdrant/docs/tree/master/qdrant/v0.11.x/"
|
||||
githubCloudDocPrefix = "https://github.com/qdrant/docs/tree/master/cloud/v0.1.x/"
|
||||
|
||||
docVersion = "v0.11.x"
|
||||
cloudDocVersion = "v0.1.x"
|
||||
|
||||
googleTagManager = "GTM-KRLCXD5"
|
||||
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
---
|
||||
title: Qdrant Cloud
|
||||
section_title: How Qdrant Cloud operates
|
||||
subtitle:
|
||||
---
|
||||
|
||||
|
||||
Qdrant Cloud is an official cloud-based managed solution by the creators of the [Qdrant](https://github.com/qdrant/qdrant) Vector Search Engine.
|
||||
It provides the same fast and reliable similarity search engine, but without a need to maintain your own infrastructure.
|
||||
|
||||
The transition from the on-premise to the cloud version of Qdrant does not require changing anything in the way you interact with the service, except for an API key that has to be provided to each request.
|
||||
|
||||
The transition is even easier if you use the official client libraries.
|
||||
|
||||
For example, the Python Qdrant client has the support of the API key already built-in, so you only need to provide it once, when the QdrantClient instance is created.
|
||||
|
||||
Please see Authentication section for details.
|
||||
@@ -0,0 +1,34 @@
|
||||
---
|
||||
title: Backups
|
||||
weight: 20
|
||||
---
|
||||
|
||||
There are situations where you need to restore your cluster because of application or system failure.
|
||||
In most cases you will have a source of truth for your data in a regular database and would be able to reindex the data into your Qdrant vector search cluster.
|
||||
However, encoding and uploading a big amount of data might require a long time.
|
||||
For high availability critical projects we highly recommend relying on replication, which is always a better option, because it guarantees the proper cluster functionality as long as at least one replica is running.
|
||||
For less critical use-cases you can make use of one of the available options.
|
||||
|
||||
## Self-service backups
|
||||
|
||||
Qdrant engine offers a snapshot API that allows to create a snapshot of a particular collection or even the whole storage.
|
||||
Please refer to the [snapshot documentation](../../snapshots/) for details.
|
||||
|
||||
In case of a single node cluster, simply call the snapshot endpoint on the exposed url.
|
||||
In case of a multi node cluster you’d need to take a snapshot on each node that the collection resides upon.
|
||||
To achieve this, you simply prepend `node-{num}-` to your cluster url and call the snapshot endpoint on the individual hosts, starting with node 0 up to the number of nodes minus one.
|
||||
You can recover snapshots the same way via api.
|
||||
|
||||
|
||||
## Automatic backups
|
||||
|
||||
**Note: not available in the beta version.**
|
||||
|
||||
Note: not available in the beta version.
|
||||
The cloud platform offers an option for automatic system backups.
|
||||
It is possible to configure periodical system level snapshots to restore a cluster from a hard copy.
|
||||
On the cluster settings section you can choose how often a backup should be done and how many latest copies should be kept on the backup storage.
|
||||
To restore a Qdrant cluster from backup, you can select a desired backup copy version and start the reporting process.
|
||||
Attention: during the restoring process the affected cluster will not be available because the cluster will be deleted and created from scratch from the backup copy.
|
||||
Please also note, that if you changed the cluster topology after the copy was created, the new cluster will reset to the previous configuration.
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
---
|
||||
title: Capacity and sizing
|
||||
weight: 20
|
||||
---
|
||||
|
||||
|
||||
We have been asked a lot about the optimal cluster configuration to serve a number of vectors.
|
||||
The only right answer is “It depends”.
|
||||
|
||||
It depends on a number of factors and options you can choose for your collections.
|
||||
|
||||
## Basic configuration
|
||||
|
||||
If you need to keep all vectors in memory for maximum performance, there is a very rough formula for estimating the needed memory size looks like this:
|
||||
|
||||
```
|
||||
memory_size = number_of_vectors * vector_dimension * 4 bytes * 1.5
|
||||
```
|
||||
|
||||
Extra 50% is needed for metadata (indexes, point versions, etc.) as well as for temporary segments constructed during the optimization process.
|
||||
|
||||
If you need to have payloads along with the vectors, it is recommended to store it on the disc, and only keep [indexed fields](../../indexing/#payload-index) in RAM.
|
||||
Read more about the payload storage in the [Storage](../../storage/#payload-storage) section.
|
||||
|
||||
|
||||
## Storage focused configuration
|
||||
|
||||
If your priority is to serve large amount of vectors with an average search latency, it is recommended to configure [mmap storage](../../storage/#configuring-memmap-storage).
|
||||
In this case vectors will be stored on the disc in memory-mapped files, and only the most frequently used vectors will be kept in RAM.
|
||||
|
||||
The amount of available RAM will significantly affect the performance of the search.
|
||||
As a rule of thumb, if you keep 2 times less vectors in RAM, the search latency will be 2 times higher.
|
||||
|
||||
The speed of disks is also important. [Let us know](mailto:cloud@qdrant.io) if you have special requirements for a high-volume search.
|
||||
|
||||
## Sub-groups oriented configuration
|
||||
|
||||
|
||||
If your use case assumes that the vectors are split into multiple collections or sub-groups based on payload values,
|
||||
it is recommended to configure memory-map storage.
|
||||
For example, if you serve search for multiple users, but each of them has an subset of vectors which they use independently.
|
||||
|
||||
In this scenatio only the active subset of vectors will be kept in RAM, which allows
|
||||
the fast search for the most active and recent users.
|
||||
|
||||
In this case you can estimate required memory size as follows:
|
||||
|
||||
```
|
||||
memory_size = number_of_active_vectors * vector_dimension * 4 bytes * 1.5
|
||||
```
|
||||
@@ -0,0 +1,58 @@
|
||||
---
|
||||
title: Quick Start in Cloud
|
||||
weight: 10
|
||||
---
|
||||
|
||||
## Create cluster
|
||||
|
||||
To start working with the Qdrant cloud platform, you have to create at least one cluster. You can do so on the clusters overview section by clicking the “Create” button.
|
||||
You will need to choose the initial configuration for your cluster as the next step.
|
||||
|
||||
|
||||
* Define the name of your cluster.
|
||||
* Choose the cloud platform provider. During the beta phase, only the AWS platform will be available for cluster deployment. GCP and Azure cloud providers will be added soon. If you have special requirements, [let us know](mailto:cloud@qdrant.io).
|
||||
* Choose the data center region. During the beta phase, only the us-east data center location will be choosable in the selection. Other cloud regions can be activated on demand. [Let us know](mailto:cloud@qdrant.io) if you have latency concerns or other topology-related requirements.
|
||||
* Choose the memory size for a node. 2GB to 64GB options are available. Please refer to the [Capacity](...........) section to make the right choice here. If you need an even higher capacity per node, [let us know](mailto:cloud@qdrant.io), we can provide machines of any size.
|
||||
* Choose the number CPU’s per node. 0.5 to 16 CPU options are available, whereas the maximal and minimal number of CPU’s is coupled to the chosen memory size.
|
||||
* And finally, choose the number of nodes you want the cluster to be deployed on. Each node is automatically attached with a disc space offering enough space for your data if you decide to put the metadata or even the index on the disc storage.
|
||||
|
||||
|
||||
### Free tier
|
||||
|
||||
Every account is eligible for one free tier cluster.
|
||||
Just choose the name, use the default configuration with 1GB memory and 0.5 CPU, and you are good to go.
|
||||
You can use this cluster for testing purposes. The capacity should be enough to serve up to 1M vectors of 768dim, but [it depends…](...........).
|
||||
|
||||
## Authentication
|
||||
|
||||
Have you created your first Qdrant cloud cluster?
|
||||
Alright, now you want to access it from within your application.
|
||||
Jump to the “Access” section, there, you will see a list of all available API keys.
|
||||
Create a new one for your newly created cluster by choosing the name from the cluster name from the select menu.
|
||||
|
||||
You can also create a key that provides access to several clusters.
|
||||
The secret key is only shown once after creation and if you lose it, you will need to create a new one. We recommend rotating the keys from time to time anyway.
|
||||
|
||||
The API key needs to be present in the request header each time you make a request to the Qdrant cloud cluster via Rest or gRPC interface.
|
||||
All official Qdrant clients for Python, Go, and Rust are supporting the API key parameter.
|
||||
|
||||
<!---
|
||||
Examples with clients
|
||||
-->
|
||||
|
||||
|
||||
```python
|
||||
from qdrant_client import QdrantClient
|
||||
|
||||
qdrant_client = QdrantClient(
|
||||
host="xyz-example.eu-central.aws.staging-cloud.qdrant.io",
|
||||
prefer_grpc=True,
|
||||
api_key="<<-provide-your-own-key->>",
|
||||
)
|
||||
```
|
||||
|
||||
```bash
|
||||
curl \
|
||||
-X GET https://xyz-example.eu-central.aws.staging-cloud.qdrant.io:6333 \
|
||||
--header 'api-key: <provide-your-own-key>'
|
||||
```
|
||||
@@ -0,0 +1,36 @@
|
||||
---
|
||||
title: Cluster scaling
|
||||
weight: 30
|
||||
---
|
||||
|
||||
|
||||
The amount of data is always growing and at some point you might need to upgrade the capacity of your cluster.
|
||||
There are different options for how it can be done.
|
||||
|
||||
## Vertical scaling
|
||||
|
||||
Vertical scaling, also known as vertical expansion, is the process of increasing the capacity of a cluster by adding more resources, such as memory, storage, or processing power.
|
||||
|
||||
You can start with a minimal cluster configuration of 2GB of RAM and resize it up to 64GB of RAM (or even more if desired) over the time step by step with the growing amount of data in your application.
|
||||
If your cluster consists of several nodes each node will need to be scaled to the same size.
|
||||
Please note that vertical cluster scaling will require a short downtime period to restart your cluster.
|
||||
In order to avoid downtime you can make use of the data replication option.
|
||||
During the beta phase, vertical scaling is available on demand and will be added as self service on the Cloud dashboard after the testing phase.
|
||||
|
||||
## Horizontal scaling
|
||||
|
||||
Vertical scaling can be an effective way to improve the performance of a cluster and extend the capacity, but it has some limitations.
|
||||
The main disadvantage of vertical scaling is that there are limits to how much a cluster can be expanded.
|
||||
At some point, adding more resources to a cluster can become impractical or cost-prohibitive.
|
||||
In such cases, horizontal scaling may be a more effective solution.
|
||||
Horizontal scaling, also known as horizontal expansion, is the process of increasing the capacity of a cluster by adding more nodes and distributing the load and data among them.
|
||||
The horizontal scaling at Qdrant starts on the collection level.
|
||||
You have to choose the number of shards you want to distribute your collection around while creating the collection.
|
||||
Please refer to the [sharding documentation](../../distributed_deployment/#sharding) section for details.
|
||||
|
||||
|
||||
Important: The number of shards means the maximum amount of nodes you can add to your cluster. In the beginning, all the shards can reside on one node.
|
||||
With the growing amount of data you can add nodes to your cluster and move shards to the dedicated nodes using the [cluster setup API](../../distributed_deployment/#cluster-scaling).
|
||||
|
||||
We, the Qdrant team, will be glad to consult you on an optimal strategy for scaling.
|
||||
[Let us know](mailto:cloud@qdrant.io) your needs and decide together on a proper solution. We plan to introduce an auto-scaling functionality. Since it is one of most desired features, it has a high priority on our Cloud roadmap.
|
||||
@@ -48,6 +48,19 @@ In addition, you can use mmap storage not only for vectors, but also for HNSW in
|
||||
To enable this, you need to set the `hnsw_config.on_disk` parameter to `true` during [creation](../collections/#create-collection) of the collection.
|
||||
|
||||
|
||||
### Configuring Memmap storage
|
||||
|
||||
To configure usage of mmap storage, you need to specify the threshold after which the segment will be converted to mmap storage.
|
||||
There are two ways to do this:
|
||||
|
||||
1. You can set the threshold globally in the [configuration file](../configuration/). The parameter is called `memmap_threshold_kb`.
|
||||
2. You can set the threshold for each collection separately during [creation](../collections/#create-collection) or [update](../collections/#update-collection-parameters).
|
||||
|
||||
|
||||
In addition, you can use mmap storage not only for vectors, but also for HNSW index.
|
||||
To enable this, you need to set the `hnsw_config.on_disk` parameter to `true` during [creation](../collections/#create-collection) of the collection.
|
||||
|
||||
|
||||
## Payload storage
|
||||
|
||||
Qdrant supports two types of payload storages: InMemory and OnDisk.
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
{{ define "main" }}
|
||||
{{ $isCloud := cond (in .Parent.RelPermalink "cloud") true false }}
|
||||
|
||||
<div class="col-12 col-md-8 col-11 pb-5">
|
||||
<div style="overflow: auto">
|
||||
<span style="float: right">
|
||||
<a href="{{ .Site.Params.githubDocPrefix }}{{ .File.LogicalName }}" target="_blank" >Edit on GitHub <i class="flaticon-github"></i></a>
|
||||
<a href="{{if $isCloud}}{{ .Site.Params.githubCloudDocPrefix }}{{else}}{{ .Site.Params.githubDocPrefix }}{{end}}{{ .File.LogicalName }}" target="_blank">Edit on GitHub <i class="flaticon-github"></i></a>
|
||||
</span>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
<ul class="breadcrumbs">
|
||||
<li class="breadcrumbs__crumb"><a href="/"><i class="flaticon-home small"></i> Home</a></li>
|
||||
{{ $rellink := "" }}
|
||||
{{ $paths := (split .RelPermalink "/") }}
|
||||
{{ $len := sub (len $paths) 2 }}
|
||||
{{ range first $len $paths }}
|
||||
{{ if gt (len . ) 0 }}
|
||||
{{ $rellink = printf "%s/%s" $rellink . }}
|
||||
/
|
||||
<li class="breadcrumbs__crumb"><a href="/{{ . }}">{{ humanize . }}</a></li>
|
||||
<li class="breadcrumbs__crumb"><a href="{{$rellink}}">{{ humanize . }}</a></li>
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
/
|
||||
|
||||
@@ -1,13 +1,23 @@
|
||||
{{$currentNode := .}}
|
||||
<nav class="col-lg-3 col-md-3 col-12 mt-0 mt-md-4 mr-3 mb-4 position-relative">
|
||||
<nav class="col-lg-3 col-md-3 col-12 mt-0 mt-md-3 pr-4 mr-3 mb-4 position-relative">
|
||||
|
||||
{{ $sectionLink := cond .IsSection .RelPermalink .Parent.RelPermalink }}
|
||||
{{ $subject := site.GetPage $sectionLink }}
|
||||
<div id="accordion" class="sticky-top documentation__sidebar">
|
||||
<!-- Button trigger modal -->
|
||||
<button type="button" class="qdr-search-input-btn w-100 mb-2" data-target="#searchModal">
|
||||
|
||||
<!-- buttons for Qdrant/Cloud doc switching-->
|
||||
{{ $isCloud := cond (in $sectionLink "cloud") true false }}
|
||||
<div class="btn-group btn-group-sm w-100 mb-3" role="group" aria-label="...">
|
||||
<a class="btn {{if $isCloud}}btn--light{{else}}btn--mid-grey font-weight-bold{{end}}" href="/documentation/">Qdrant</a>
|
||||
<a class="btn {{if $isCloud}}btn--mid-grey font-weight-bold{{else}}btn--light{{end}}" href="/documentation/cloud/">Cloud</a>
|
||||
</div>
|
||||
<!-- Button triggers search modal -->
|
||||
<button type="button" class="qdr-search-input-btn w-100 mb-2 w-100" data-target="#searchModal">
|
||||
<i class="fa fa-search mr-2"></i>Search...
|
||||
</button>
|
||||
|
||||
<div class="documentation__sidebar-list">
|
||||
{{ range where .Site.RegularPages "Section" .Section }}
|
||||
<div class="documentation__sidebar-list">
|
||||
{{ range $subject.RegularPages }}
|
||||
<div class="link-group {{ if eq .File.UniqueID $currentNode.File.UniqueID }}active {{ end }}">
|
||||
<div class="link-group-heading" id="headingOne">
|
||||
<span class="mb-0">
|
||||
|
||||
@@ -91,6 +91,24 @@
|
||||
&.disabled {
|
||||
display: none;
|
||||
}
|
||||
|
||||
// backgrounds
|
||||
&.btn--primary {
|
||||
background-color: var(--brand-primary);
|
||||
color: var(--white);
|
||||
}
|
||||
&.btn--secondary {
|
||||
background-color: var(--brand-secondary);
|
||||
color: var(--white);
|
||||
}
|
||||
&.btn--light {
|
||||
background-color: var(--brand-light);
|
||||
color: var(--brand-secondary);
|
||||
}
|
||||
&.btn--mid-grey {
|
||||
background-color: var(--brand-mid-grey);
|
||||
color: var(--brand-secondary);
|
||||
}
|
||||
}
|
||||
|
||||
.fixed-header .btn_in-header {
|
||||
|
||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user