-
Notifications
You must be signed in to change notification settings - Fork 3
Expand file tree
/
Copy pathmultiframe_demo.py
More file actions
77 lines (66 loc) · 2.46 KB
/
multiframe_demo.py
File metadata and controls
77 lines (66 loc) · 2.46 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
import base64
from openai import OpenAI
from system_prompt import steve_system_prompt
instruction = "User task: search for today's weather. Please generate the next move."
gt_response = f.read().strip()
client = OpenAI(
base_url="http://127.0.0.1:8000/v1",
api_key="empty",
)
with open(screenshot1_path, 'rb') as image1_file:
screenshot1_string = base64.b64encode(image1_file.read()).decode("utf-8")
with open(screenshot2_path, 'rb') as image2_file:
screenshot2_string = base64.b64encode(image2_file.read()).decode("utf-8")
round1_response = client.chat.completions.create(
model="cot_qwen2vl",
messages=[
{
"role": "user",
"content": [
{"type": "text", "text": steve_system_prompt},
{"type": "image_url", "image_url": {"url": f"data:image/png;base64,{screenshot1_string}"}},
{"type": "text", "text": instruction}
],
},
],
frequency_penalty=0.2,
temperature=0.6,
max_tokens=4096,
extra_body={"skip_special_tokens": False}
)
round1_response = round1_response.choices[0].message.content
# Here we use the ground truth response for the second round
round1_response = gt_response
round2_response = client.chat.completions.create(
model="cot_qwen2vl",
messages=[
{
"role": "user",
"content": [
{"type": "text", "text": steve_system_prompt},
{"type": "image_url", "image_url": {"url": f"data:image/png;base64,{screenshot1_string}"}},
{"type": "text", "text": instruction}
],
},
{
"role": "assistant",
"content": round1_response,
},
{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": f"data:image/png;base64,{screenshot2_string}"}},
{"type": "text", "text": instruction}
]
}
],
frequency_penalty=0.2,
temperature=0.6,
max_tokens=4096,
extra_body={"skip_special_tokens": False}
)
round2_response = round2_response.choices[0].message.content
print(round2_response)