landing_page/qdrant-landing/content/documentation/headless/snippets/time-based-sharding/go.go

package snippet

import (
	"context"
	"encoding/csv"
	"fmt"
	"io"
	"net/http"
	"time"

	"github.com/google/uuid"
	"github.com/qdrant/go-client/qdrant"
)

// @block-start parse-csv
type CSVRow struct {
	Text     string
	Datetime string
}

func parseCSV(url string, fn func(CSVRow)) error {
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	csvReader := csv.NewReader(resp.Body)
	headers, err := csvReader.Read()
	if err != nil {
		return err
	}

	textIdx, datetimeIdx := -1, -1
	for i, h := range headers {
		switch h {
		case "text":
			textIdx = i
		case "datetime":
			datetimeIdx = i
		}
	}

	for {
		row, err := csvReader.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		fn(CSVRow{Text: row[textIdx], Datetime: row[datetimeIdx]})
	}
	return nil
}
// @block-end parse-csv

func Main() {
	// @hide-start
	QDRANT_URL := ""
	QDRANT_API_KEY := ""
	// @hide-end

	// @block-start initialize-client
	client, err := qdrant.NewClient(&qdrant.Config{
		Host:   QDRANT_URL,
		APIKey: QDRANT_API_KEY,
		UseTLS: true,
	})
	// @block-end initialize-client

	// @hide-start
	if err != nil {
		panic(err)
	}
	// @hide-end

	// @block-start create-collection
	collectionName := "my_collection"

	exists, err := client.CollectionExists(context.Background(), collectionName)
	if err != nil { panic(err) } // @hide
	if exists {
		client.DeleteCollection(context.Background(), collectionName)
	}

	client.CreateCollection(context.Background(), &qdrant.CreateCollection{
		CollectionName: collectionName,
		VectorsConfig: qdrant.NewVectorsConfigMap(
			map[string]*qdrant.VectorParams{
				"dense_vector": {
					Size:     384,
					Distance: qdrant.Distance_Cosine,
				},
			},
		),
		ShardingMethod: qdrant.ShardingMethod_Custom.Enum(),
	})
	// @block-end create-collection

	// @block-start upload-vectors
	csvUrl := "https://raw.githubusercontent.com/qdrant/examples/refs/heads/master/time-based-sharding/social-media-posts.csv"

	shardKeyDescriptions, err := client.ListShardKeys(context.Background(), collectionName)
	if err != nil { panic(err) } // @hide

	// Retrieve a list of existing shard keys in the collection
	existingShardKeys := make(map[string]bool)
	for _, desc := range shardKeyDescriptions {
		existingShardKeys[desc.Key.GetKeyword()] = true
	}

	denseModel := "sentence-transformers/all-MiniLM-L6-v2"
	batchSize := 100
	var currentDate string
	var buffer []*qdrant.PointStruct

	err = parseCSV(csvUrl, func(row CSVRow) {
		text := row.Text
		datetime := row.Datetime
		shardDate := datetime[:10] // Extract YYYY-MM-DD

		if shardDate != currentDate {
			// Flush buffer for the previous date before switching
			if len(buffer) > 0 {
				client.Upsert(context.Background(), &qdrant.UpsertPoints{
					CollectionName: collectionName,
					Points:         buffer,
					ShardKeySelector: &qdrant.ShardKeySelector{
						ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey(currentDate)},
					},
				})
				buffer = nil
			}

			// Create shard for the new date if it doesn't exist yet
			if !existingShardKeys[shardDate] {
				client.CreateShardKey(context.Background(), collectionName, &qdrant.CreateShardKey{
					ShardKey: qdrant.NewShardKey(shardDate),
				})
				existingShardKeys[shardDate] = true
			}

			currentDate = shardDate
		}

		// Add point to buffer
		buffer = append(buffer, &qdrant.PointStruct{
			Id: qdrant.NewID(uuid.New().String()),
			Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{
				"dense_vector": qdrant.NewVectorDocument(&qdrant.Document{
					Text:  text,
					Model: denseModel,
				}),
			}),
			Payload: qdrant.NewValueMap(map[string]any{
				"text":     text,
				"datetime": datetime,
			}),
		})

		// Flush batch if buffer size exceeds batch size
		if len(buffer) >= batchSize {
			client.Upsert(context.Background(), &qdrant.UpsertPoints{
				CollectionName: collectionName,
				Points:         buffer,
				ShardKeySelector: &qdrant.ShardKeySelector{
					ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey(currentDate)},
				},
			})
			buffer = nil
		}
	})
	if err != nil { panic(err) } // @hide

	// Flush remaining partial batch
	if len(buffer) > 0 {
		client.Upsert(context.Background(), &qdrant.UpsertPoints{
			CollectionName: collectionName,
			Points:         buffer,
			ShardKeySelector: &qdrant.ShardKeySelector{
				ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey(currentDate)},
			},
		})
	}
	// @block-end upload-vectors

	// @block-start search-single-shard
	queryText := "coffee"

	result, err := client.Query(context.Background(), &qdrant.QueryPoints{
		CollectionName: collectionName,
		Query:          qdrant.NewQueryDocument(&qdrant.Document{Text: queryText, Model: denseModel}),
		Using:          qdrant.PtrOf("dense_vector"),
		Limit:          qdrant.PtrOf(uint64(5)),
		ShardKeySelector: &qdrant.ShardKeySelector{
			ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey("2026-04-07")},
		},
	})

	// @hide-start
	if err != nil {
		panic(err)
	}
	// @hide-end

	for _, hit := range result {
		fmt.Println(hit)
	}
	// @block-end search-single-shard

	// @block-start search-multiple-shards
	result, err = client.Query(context.Background(), &qdrant.QueryPoints{
		CollectionName: collectionName,
		Query:          qdrant.NewQueryDocument(&qdrant.Document{Text: queryText, Model: denseModel}),
		Using:          qdrant.PtrOf("dense_vector"),
		Limit:          qdrant.PtrOf(uint64(5)),
		ShardKeySelector: &qdrant.ShardKeySelector{
			ShardKeys: []*qdrant.ShardKey{
				qdrant.NewShardKey("2026-04-06"),
				qdrant.NewShardKey("2026-04-07"),
			},
		},
	})

	// @hide-start
	if err != nil {
		panic(err)
	}
	// @hide-end

	for _, hit := range result {
		fmt.Println(hit)
	}
	// @block-end search-multiple-shards

	// @block-start search-all-shards
	result, err = client.Query(context.Background(), &qdrant.QueryPoints{
		CollectionName: collectionName,
		Query:          qdrant.NewQueryDocument(&qdrant.Document{Text: queryText, Model: denseModel}),
		Using:          qdrant.PtrOf("dense_vector"),
		Limit:          qdrant.PtrOf(uint64(5)),
	})

	// @hide-start
	if err != nil {
		panic(err)
	}
	// @hide-end

	for _, hit := range result {
		fmt.Println(hit)
	}
	// @block-end search-all-shards

	// @block-start pruning-shards
	today := "2026-04-08"
	t, _ := time.Parse("2006-01-02", today)
	oldestShardKey := t.AddDate(0, 0, -7).Format("2006-01-02")

	client.CreateShardKey(context.Background(), collectionName, &qdrant.CreateShardKey{
		ShardKey: qdrant.NewShardKey(today),
	})
	client.DeleteShardKey(context.Background(), collectionName, &qdrant.DeleteShardKey{
		ShardKey: qdrant.NewShardKey(oldestShardKey),
	})
	// @block-end pruning-shards

	// @block-start ingest-new-data
	client.Upsert(context.Background(), &qdrant.UpsertPoints{
		CollectionName: collectionName,
		Points: []*qdrant.PointStruct{
			{
				Id: qdrant.NewID(uuid.New().String()),
				Vectors: qdrant.NewVectorsMap(map[string]*qdrant.Vector{
					"dense_vector": qdrant.NewVectorDocument(&qdrant.Document{
						Text:  "The best way to start a Wednesday is with a cup of coffee",
						Model: denseModel,
					}),
				}),
				Payload: qdrant.NewValueMap(map[string]any{
					"text":     "The best way to start a Wednesday is with a cup of coffee",
					"datetime": "2026-04-08T07:57:47",
				}),
			},
		},
		ShardKeySelector: &qdrant.ShardKeySelector{
			ShardKeys: []*qdrant.ShardKey{qdrant.NewShardKey(today)},
		},
	})
	// @block-end ingest-new-data
}