krzysztof/landing_page
1
0
Fork 0
mirror of https://github.com/qdrant/landing_page.git synced 2026-09-26 22:48:30 +02:00
Code Issues Packages Projects Releases Wiki Activity

Snipets tooling update

Browse Source 
- update README
- generate-md.py: ignore trailing newlines when complaining about diffs
- all langs: add more dependencies to templates
- all langs: support @hide-start and @hide-end for multi-line hiding
- go: also allow one-line imports
- java: allow declarations other than `main` method
- python, typescript: implement `unshorten` for implicit imports and
  client initialization
- python: add mypy config as `datasets` lacks type py.typed
- python: hide `# mypy:` annotations
- typescript: add tsconfig that allows top-level `await`s
- typescript: depend on @types/node to fix `import ... from "crypto"`
This commit is contained in:
xzfc
2025-11-28 21:57:13 +00:00
parent 8d6772736d
commit cdfbd6538a
18 changed files with 1217 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
automation/snippets/README.md
+9
View File
@@ -114,3 +114,12 @@ Each supported language has:
This directory is gitignored.
- For Go, Rust, Python it updates the lockfile in the [`templates/`](./templates) directory.
These lockfiles are checked in this repo.
## Quirks
Sometimes `mypy` (python typechecker) complains at valid code.
Place this comment at the top of the file to silence it:
```python
# mypy: disable-error-code="arg-type"
```
automation/snippets/generate-md.py
+3 -3
View File
@@ -52,11 +52,11 @@ def main() -> None:
snippet_dir / f"{lang.NAME}.md",
)
handwritten = (snippet_dir / f"{lang.NAME}.md").read_text()
if handwritten != generated:
if handwritten.rstrip("\n") != generated.rstrip("\n"):
print_and_colorize_diff(
difflib.unified_diff(
handwritten.splitlines(keepends=True),
generated.splitlines(keepends=True),
handwritten.rstrip("\n").splitlines(keepends=True),
generated.rstrip("\n").splitlines(keepends=True),
fromfile=str(handwritten_fname),
tofile=str(generated_fname),
),
automation/snippets/lib/languages/base.py
+30 -8
View File
@@ -81,7 +81,7 @@ def template(
target_fname.write_text("".join(result))
_RE_COMMENT = re.compile(r".*\s(?://|#)\s*(@.*)$")
_RE_COMMENT = re.compile(r"^(.*\s|)(?://|#)\s*(@.*)$")
def generic_shorten(text: str) -> str:
@@ -90,14 +90,36 @@ def generic_shorten(text: str) -> str:
Removes comments with @hide annotation and trims excessive newlines.
"""
result = []
hide_mode = False
for line in text.splitlines():
if (m := _RE_COMMENT.match(line)) is not None:
if m[1] == "@hide":
continue
else:
msg = f"Unknown annotation: {m[1]}"
raise ValueError(msg)
result.append(line + "\n")
if (m := _RE_COMMENT.match(line)) is None:
if not hide_mode:
result.append(line + "\n")
continue
has_code = m[1].strip() != ""
annotation = m[2]
if annotation == "@hide":
if not has_code:
raise ValueError("Hiding empty line is not allowed")
if hide_mode:
raise ValueError("@hide inside @hide-start/@hide-end is not allowed")
elif annotation == "@hide-start":
if has_code:
raise ValueError("@hide-start should be on its own line")
if hide_mode:
raise ValueError("Nesting @hide-start is not allowed")
hide_mode = True
elif annotation == "@hide-end":
if has_code:
raise ValueError("@hide-end should be on its own line")
if not hide_mode:
raise ValueError("@hide-end without matching @hide-start")
hide_mode = False
else:
raise ValueError(f"Unknown annotation: {m[1]}")
if hide_mode:
raise ValueError("Unclosed @hide-start")
text = "".join(result)
text = text.lstrip("\n").rstrip("\n")
text = re.sub(r"\n{3,}", "\n\n", text)
automation/snippets/lib/languages/go.py
+6 -1
View File
@@ -92,7 +92,12 @@ class LanguageGo(Language):
RE_RENDERED = re.compile(
r"""
(?P<imports> (?:import\s*\([^)]+\)\n|\n)* )
(?P<imports>
(?: import\s*\([^)]+\)\n
| import\s+"[^"]+"\n
| \n
)*
)
(?P<body> .* )
$
""",
automation/snippets/lib/languages/java.py
+10 -1
View File
@@ -65,6 +65,7 @@ class LanguageJava(Language):
package\s[a-zA-Z0-9_.]+;\n
(?P<imports> .*? )
\s*public\ class\ Snippet\ \{\n
(?P<methods> .*? )
\s*public\ static\ void\ run\(\)\ throws\ Exception\ \{\n
(?P<body> .*? )
\s*\}\s*}\s*
@@ -80,6 +81,10 @@ class LanguageJava(Language):
import io.qdrant.client.QdrantClient;
public class Snippet {
static void additionalMethod() {
// Some additional method (optional)
}
public static void run() throws Exception {
// Your code here
}
@@ -94,7 +99,11 @@ class LanguageJava(Language):
msg = "Invalid snippet format"
raise ValueError(msg)
return generic_shorten(
m["imports"].strip() + "\n\n" + textwrap.dedent(m["body"]).strip()
m["imports"].strip()
+ "\n\n"
+ textwrap.dedent(m["methods"]).strip()
+ "\n\n"
+ textwrap.dedent(m["body"]).strip()
)
RE_RENDERED = re.compile(
automation/snippets/lib/languages/python.py
+64 -2
View File
@@ -1,7 +1,19 @@
import re
import subprocess
from pathlib import Path
from .base import CompileResult, Language, copy_template, trim_commonpath
from .base import (
CompileResult,
Language,
copy_template,
generic_shorten,
trim_commonpath,
)
RE_IMPORTS = re.compile(
r"^from qdrant_client import (.*)$",
re.MULTILINE,
)
class LanguagePython(Language):
@@ -28,7 +40,57 @@ class LanguagePython(Language):
target_path,
]
p = subprocess.run(["uv", "--project=templates/python", "run", "mypy", tmpdir])
p = subprocess.run(
[
"uv",
"--project=templates/python",
"run",
"mypy",
"--config-file=templates/python/pyproject.toml",
tmpdir,
]
)
result.has_issues = p.returncode != 0
return result
@classmethod
def shorten(cls, contents: str) -> str:
lines = [
line
for line in contents.splitlines(keepends=True)
if not line.lstrip().startswith("# mypy:")
]
return generic_shorten("".join(lines))
@classmethod
def unshorten(cls, contents: str) -> str:
if "client." in contents and "QdrantClient" not in contents:
contents = (
'client = QdrantClient(url="http://localhost:6333") # @hide\n\n'
+ contents.lstrip()
)
has_imports = set()
if m := RE_IMPORTS.search(contents):
has_imports = set(i.strip() for i in m[1].split(","))
need_imports = set()
if "QdrantClient" in contents:
need_imports.add("QdrantClient")
if "models." in contents:
need_imports.add("models")
if need_imports - has_imports:
if m is not None:
contents = RE_IMPORTS.sub(
f"from qdrant_client import {', '.join(sorted(need_imports | has_imports))}",
contents,
)
else:
contents = (
f"from qdrant_client import {', '.join(sorted(need_imports))} # @hide\n\n"
+ contents.lstrip()
)
return contents
automation/snippets/lib/languages/typescript.py
+28 -9
View File
@@ -14,20 +14,29 @@ class LanguageTypescript(Language):
def compile(cls, tmpdir: Path, fnames: list[Path]) -> CompileResult:
tmpdir.mkdir()
package_data = json.loads(Path("templates/typescript/package.json").read_text())
trimmed_fnames = trim_commonpath(fnames)
# package.json
package_data = json.loads(Path("templates/typescript/package.json").read_text())
package_data["dependencies"]["@qdrant/js-client-rest"] = "file:" + str(
Path("clients/typescript/packages/js-client-rest").resolve().absolute()
)
(tmpdir / "package.json").write_text(json.dumps(package_data, indent=2))
# package-lock.json
shutil.copyfile(
"templates/typescript/package-lock.json", f"{tmpdir}/package-lock.json"
"templates/typescript/package-lock.json",
f"{tmpdir}/package-lock.json",
)
trimmed_fnames = trim_commonpath(fnames)
result = CompileResult()
# tsconfig.json
tsconfig_data = json.loads(
Path("templates/typescript/tsconfig.json").read_text()
)
tsconfig_data["files"] = [f"s/{fname}" for fname in trimmed_fnames.values()]
(tmpdir / "tsconfig.json").write_text(json.dumps(tsconfig_data, indent=2))
result = CompileResult()
for snippet_fname, trimmed_fname in trimmed_fnames.items():
dest_path = tmpdir / "s" / trimmed_fname
copy_template(snippet_fname, dest_path)
@@ -38,15 +47,25 @@ class LanguageTypescript(Language):
result.has_issues = True
return result
p = subprocess.run(
["node_modules/.bin/tsc"]
+ ["s" / trimmed_fname for trimmed_fname in trimmed_fnames.values()],
cwd=tmpdir,
)
p = subprocess.run(["node_modules/.bin/tsc"], cwd=tmpdir)
result.has_issues = p.returncode != 0
return result
@classmethod
def unshorten(cls, contents: str) -> str:
if "client." in contents and "new QdrantClient" not in contents:
contents = (
'const client = new QdrantClient({ host: "localhost", port: 6333 }); // @hide\n\n'
+ contents.lstrip()
)
if "QdrantClient" in contents and "@qdrant/js-client-rest" not in contents:
contents = (
'import { QdrantClient } from "@qdrant/js-client-rest"; // @hide\n\n'
+ contents.lstrip()
)
return contents
@classmethod
def format(cls, fnames: list[str]) -> None:
# subprocess.run(["npx", "prettier", "--write", *fnames], check=True)
automation/snippets/migrate-snippet.py
+2 -1
View File
@@ -34,8 +34,9 @@ def main() -> None:
log(f"Skipping unknown language file: {md_file}")
continue
lines = md_file.read_text().rstrip().splitlines()
lines = md_file.read_text().strip().splitlines()
if not lines[0].startswith("```") or lines[-1] != "```":
log(f"Can't parse snippet file: {md_file}, skipping")
continue
content = "\n".join(lines[1:-1]) + "\n"
content = lang.unshorten(content)
automation/snippets/templates/csharp/csharp.csproj
+4
View File
@@ -6,6 +6,10 @@
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<AssemblyName>snippets-amalgamation</AssemblyName>
<NoWarn>CS1998<!--
disable warning for async method without 'await',
since all our Snippet.Run() methods are async
--></NoWarn>
</PropertyGroup>
<ItemGroup>
automation/snippets/templates/go/go.mod
+4 -1
View File
@@ -2,7 +2,10 @@ module example.com/snippets-amalgamation
go 1.25.2
require github.com/qdrant/go-client v1.16.0
require (
github.com/google/uuid v1.6.0
github.com/qdrant/go-client v1.16.0
)
require (
golang.org/x/net v0.47.0 // indirect
automation/snippets/templates/go/main.go
+1
View File
@@ -2,6 +2,7 @@ package main
import (
"fmt"
_ "github.com/google/uuid"
_ "github.com/qdrant/go-client/qdrant"
"os"
// %imports%
automation/snippets/templates/python/pyproject.toml
+5
View File
@@ -3,6 +3,7 @@ name = "snippets-amalgamation"
version = "0.1.0"
requires-python = ">=3.12"
dependencies = [
"datasets>=4.4.1",
"qdrant-client",
]
@@ -13,3 +14,7 @@ qdrant-client = { git = "https://github.com/qdrant/qdrant-client", tag = "v1.16.
dev = [
"mypy>=1.18.2",
]
[[tool.mypy.overrides]]
module = ["datasets"]
ignore_missing_imports = true
automation/snippets/templates/python/uv.lock
Generated
+1005 -1
View File
File diff suppressed because it is too large Load Diff
automation/snippets/templates/rust/Cargo.lock
Generated
+13
View File
@@ -1359,7 +1359,9 @@ version = "0.1.0"
dependencies = [
"anyhow",
"qdrant-client",
"serde_json",
"tokio",
"uuid",
]
[[package]]
@@ -1722,6 +1724,17 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
[[package]]
name = "uuid"
version = "1.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2"
dependencies = [
"getrandom 0.3.4",
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "want"
version = "0.3.1"
automation/snippets/templates/rust/Cargo.toml
+2
View File
@@ -6,4 +6,6 @@ edition = "2024"
[dependencies]
anyhow = "1.0.100"
qdrant-client = { git = "https://github.com/qdrant/rust-client", branch = "master" }
serde_json = "1.0.145"
tokio = { version = "1.48.0", features = ["rt-multi-thread", "macros"] }
uuid = { version = "1.18.1", features = ["v4"] }
automation/snippets/templates/typescript/package-lock.json
Generated
+19 -1
View File
@@ -12,6 +12,7 @@
"@qdrant/js-client-rest": "file:../../clients/typescript/packages/js-client-rest"
},
"devDependencies": {
"@types/node": "^24.10.1",
"typescript": "^5.9.3"
}
},
@@ -20,7 +21,7 @@
},
"../../clients/typescript/packages/js-client-rest": {
"name": "@qdrant/js-client-rest",
"version": "1.16.0",
"version": "1.16.1",
"license": "Apache-2.0",
"dependencies": {
"@qdrant/openapi-typescript-fetch": "1.2.6",
@@ -49,6 +50,16 @@
"resolved": "../../clients/typescript/packages/js-client-rest",
"link": true
},
"node_modules/@types/node": {
"version": "24.10.1",
"resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.1.tgz",
"integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"undici-types": "~7.16.0"
}
},
"node_modules/typescript": {
"version": "5.9.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
@@ -62,6 +73,13 @@
"engines": {
"node": ">=14.17"
}
},
"node_modules/undici-types": {
"version": "7.16.0",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz",
"integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==",
"dev": true,
"license": "MIT"
}
}
}
automation/snippets/templates/typescript/package.json
+2
View File
@@ -1,6 +1,7 @@
{
"name": "snippets-amalgamation",
"version": "1.0.0",
"type": "module",
"description": "",
"main": "1.js",
"scripts": {
@@ -9,6 +10,7 @@
"author": "",
"license": "ISC",
"devDependencies": {
"@types/node": "^24.10.1",
"typescript": "^5.9.3"
},
"dependencies": {
automation/snippets/templates/typescript/tsconfig.json
+10
View File
@@ -0,0 +1,10 @@
{
"compilerOptions": {
"target": "ES2022",
"module": "esnext",
"moduleResolution": "bundler",
"esModuleInterop": true,
"skipLibCheck": true,
"strict": true
}
}