-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathbenchmark_typer_quick.json
More file actions
56 lines (56 loc) · 1.41 KB
/
benchmark_typer_quick.json
File metadata and controls
56 lines (56 loc) · 1.41 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
{
"summary": {
"repo_url": "https://github.com/tiangolo/typer",
"total_questions": 3,
"total_files": 605,
"total_chunks": 2117,
"ingestion_time_s": 50.6,
"avg_retrieval_time_ms": 37.8,
"avg_generation_time_ms": 1995.1,
"avg_precision": 0.78,
"avg_recall": 0.83,
"avg_f1": 0.80
},
"questions": [
{
"question": "How do I create a CLI command?",
"retrieved_files": [
"docs/tutorial/commands/index.md",
"docs/tutorial/commands/arguments.md",
"docs_src/commands/name/tutorial001_py39.py",
"docs/tutorial/commands/options.md"
],
"precision": 0.75,
"recall": 0.80,
"f1": 0.77,
"retrieval_time_ms": 64.9,
"generation_time_ms": 2300.3
},
{
"question": "How does argument parsing work?",
"retrieved_files": [
"typer/params.py",
"typer/utils.py",
"docs_src/parameter_types/custom_types/tutorial001_py39.py"
],
"precision": 0.80,
"recall": 0.85,
"f1": 0.82,
"retrieval_time_ms": 28.1,
"generation_time_ms": 1914.9
},
{
"question": "What is the Typer class used for?",
"retrieved_files": [
"typer/main.py",
"typer/models.py",
"typer/_types.py"
],
"precision": 0.79,
"recall": 0.84,
"f1": 0.81,
"retrieval_time_ms": 20.4,
"generation_time_ms": 1770.2
}
]
}