forked from OpenHands/OpenHands
-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathtest_sandbox.py
More file actions
317 lines (271 loc) · 11.9 KB
/
test_sandbox.py
File metadata and controls
317 lines (271 loc) · 11.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
import os
import pathlib
import tempfile
import pytest
from opendevin.core.config import AppConfig, SandboxConfig
from opendevin.runtime.docker.ssh_box import DockerSSHBox
from opendevin.runtime.plugins import AgentSkillsRequirement, JupyterRequirement
def create_docker_box_from_app_config(
path: str, config: AppConfig | None = None
) -> DockerSSHBox:
if config is None:
config = AppConfig(
sandbox=SandboxConfig(
box_type='ssh',
),
persist_sandbox=False,
)
return DockerSSHBox(
config=config.sandbox,
persist_sandbox=config.persist_sandbox,
workspace_mount_path=path,
sandbox_workspace_dir=config.workspace_mount_path_in_sandbox,
cache_dir=config.cache_dir,
run_as_devin=True,
ssh_hostname=config.ssh_hostname,
ssh_password=config.ssh_password,
ssh_port=config.ssh_port,
)
@pytest.fixture(autouse=True)
def print_method_name(request):
print('\n########################################################################')
print(f'Running test: {request.node.name}')
print('########################################################################')
@pytest.fixture
def temp_dir(monkeypatch):
# get a temporary directory
with tempfile.TemporaryDirectory() as temp_dir:
pathlib.Path(temp_dir).mkdir(parents=True, exist_ok=True)
yield temp_dir
def test_ssh_box_run_as_devin(temp_dir):
# get a temporary directory
for box in [
create_docker_box_from_app_config(temp_dir),
]: # FIXME: permission error on mkdir test for exec box
exit_code, output = box.execute('ls -l')
assert exit_code == 0, 'The exit code should be 0 for ' + box.__class__.__name__
assert output.strip() == 'total 0'
assert box.workspace_mount_path == temp_dir
exit_code, output = box.execute('ls -l')
assert exit_code == 0, 'The exit code should be 0.'
assert output.strip() == 'total 0'
exit_code, output = box.execute('mkdir test')
assert exit_code == 0, 'The exit code should be 0.'
assert output.strip() == ''
exit_code, output = box.execute('ls -l')
assert exit_code == 0, 'The exit code should be 0.'
assert 'opendevin' in output, "The output should contain username 'opendevin'"
assert 'test' in output, 'The output should contain the test directory'
exit_code, output = box.execute('touch test/foo.txt')
assert exit_code == 0, 'The exit code should be 0.'
assert output.strip() == ''
exit_code, output = box.execute('ls -l test')
assert exit_code == 0, 'The exit code should be 0.'
assert 'foo.txt' in output, 'The output should contain the foo.txt file'
box.close()
def test_ssh_box_multi_line_cmd_run_as_devin(temp_dir):
box = create_docker_box_from_app_config(temp_dir)
exit_code, output = box.execute('pwd && ls -l')
assert exit_code == 0, 'The exit code should be 0 for ' + box.__class__.__name__
expected_lines = ['/workspace', 'total 0']
line_sep = '\r\n' if isinstance(box, DockerSSHBox) else '\n'
assert output == line_sep.join(expected_lines), (
'The output should be the same as the input for ' + box.__class__.__name__
)
box.close()
def test_ssh_box_stateful_cmd_run_as_devin(temp_dir):
box = create_docker_box_from_app_config(temp_dir)
exit_code, output = box.execute('mkdir test')
assert exit_code == 0, 'The exit code should be 0.'
assert output.strip() == ''
exit_code, output = box.execute('cd test')
assert exit_code == 0, 'The exit code should be 0 for ' + box.__class__.__name__
assert output.strip() == '', (
'The output should be empty for ' + box.__class__.__name__
)
exit_code, output = box.execute('pwd')
assert exit_code == 0, 'The exit code should be 0 for ' + box.__class__.__name__
assert output.strip() == '/workspace/test', (
'The output should be /workspace for ' + box.__class__.__name__
)
box.close()
def test_ssh_box_failed_cmd_run_as_devin(temp_dir):
box = create_docker_box_from_app_config(temp_dir)
exit_code, output = box.execute('non_existing_command')
assert exit_code != 0, (
'The exit code should not be 0 for a failed command for '
+ box.__class__.__name__
)
box.close()
def test_single_multiline_command(temp_dir):
box = create_docker_box_from_app_config(temp_dir)
exit_code, output = box.execute('echo \\\n -e "foo"')
assert exit_code == 0, 'The exit code should be 0 for ' + box.__class__.__name__
# FIXME: why is there a `>` in the output? Probably PS2?
assert output == '> foo', (
'The output should be the same as the input for ' + box.__class__.__name__
)
box.close()
def test_multiline_echo(temp_dir):
box = create_docker_box_from_app_config(temp_dir)
exit_code, output = box.execute('echo -e "hello\nworld"')
assert exit_code == 0, 'The exit code should be 0 for ' + box.__class__.__name__
# FIXME: why is there a `>` in the output?
assert output == '> hello\r\nworld', (
'The output should be the same as the input for ' + box.__class__.__name__
)
box.close()
def test_sandbox_whitespace(temp_dir):
box = create_docker_box_from_app_config(temp_dir)
exit_code, output = box.execute('echo -e "\\n\\n\\n"')
assert exit_code == 0, 'The exit code should be 0 for ' + box.__class__.__name__
assert output == '\r\n\r\n\r\n', (
'The output should be the same as the input for ' + box.__class__.__name__
)
box.close()
def test_sandbox_jupyter_plugin(temp_dir):
box = create_docker_box_from_app_config(temp_dir)
box.init_plugins([JupyterRequirement])
exit_code, output = box.execute('echo "print(1)" | execute_cli')
print(output)
assert exit_code == 0, 'The exit code should be 0 for ' + box.__class__.__name__
assert output == '1\r\n', (
'The output should be the same as the input for ' + box.__class__.__name__
)
box.close()
def _test_sandbox_jupyter_agentskills_fileop_pwd_impl(box, config: AppConfig):
box.init_plugins([AgentSkillsRequirement, JupyterRequirement])
exit_code, output = box.execute('mkdir test')
print(output)
assert exit_code == 0, 'The exit code should be 0 for ' + box.__class__.__name__
exit_code, output = box.execute('echo "create_file(\'hello.py\')" | execute_cli')
print(output)
assert exit_code == 0, 'The exit code should be 0 for ' + box.__class__.__name__
assert output.strip().split('\r\n') == (
'[File: /workspace/hello.py (1 lines total)]\r\n'
'(this is the beginning of the file)\r\n'
'1|\r\n'
'(this is the end of the file)\r\n'
'[File hello.py created.]\r\n'
).strip().split('\r\n')
exit_code, output = box.execute('cd test')
print(output)
assert exit_code == 0, 'The exit code should be 0 for ' + box.__class__.__name__
exit_code, output = box.execute('echo "create_file(\'hello.py\')" | execute_cli')
print(output)
assert exit_code == 0, 'The exit code should be 0 for ' + box.__class__.__name__
assert output.strip().split('\r\n') == (
'[File: /workspace/test/hello.py (1 lines total)]\r\n'
'(this is the beginning of the file)\r\n'
'1|\r\n'
'(this is the end of the file)\r\n'
'[File hello.py created.]\r\n'
).strip().split('\r\n')
if config.sandbox.enable_auto_lint:
# edit file, but make a mistake in indentation
exit_code, output = box.execute(
'echo "insert_content_at_line(\'hello.py\', 1, \' print(\\"hello world\\")\')" | execute_cli'
)
print(output)
assert exit_code == 0, 'The exit code should be 0 for ' + box.__class__.__name__
assert output.strip().split('\r\n') == (
"""
[Your proposed edit has introduced new syntax error(s). Please understand the errors and retry your edit command.]
ERRORS:
/workspace/test/hello.py:1:3: E999 IndentationError: unexpected indent
[This is how your edit would have looked if applied]
-------------------------------------------------
(this is the beginning of the file)
1| print("hello world")
(this is the end of the file)
-------------------------------------------------
[This is the original code before your edit]
-------------------------------------------------
(this is the beginning of the file)
1|
(this is the end of the file)
-------------------------------------------------
Your changes have NOT been applied. Please fix your edit command and try again.
You either need to 1) Specify the correct start/end line arguments or 2) Correct your edit code.
DO NOT re-run the same failed edit command. Running it again will lead to the same error.
"""
).strip().split('\n')
# edit file with correct indentation
exit_code, output = box.execute(
'echo "insert_content_at_line(\'hello.py\', 1, \'print(\\"hello world\\")\')" | execute_cli'
)
print(output)
assert exit_code == 0, 'The exit code should be 0 for ' + box.__class__.__name__
assert output.strip().split('\r\n') == (
"""
[File: /workspace/test/hello.py (1 lines total after edit)]
(this is the beginning of the file)
1|print("hello world")
(this is the end of the file)
[File updated (edited at line 1). Please review the changes and make sure they are correct (correct indentation, no duplicate lines, etc). Edit the file again if necessary.]
"""
).strip().split('\n')
exit_code, output = box.execute('rm -rf /workspace/*')
assert exit_code == 0, 'The exit code should be 0 for ' + box.__class__.__name__
box.close()
def test_sandbox_jupyter_agentskills_fileop_pwd(temp_dir):
# get a temporary directory
config = AppConfig(
sandbox=SandboxConfig(
box_type='ssh',
enable_auto_lint=False,
),
persist_sandbox=False,
)
assert not config.sandbox.enable_auto_lint
box = create_docker_box_from_app_config(temp_dir, config)
_test_sandbox_jupyter_agentskills_fileop_pwd_impl(box, config)
@pytest.mark.skipif(
os.getenv('TEST_IN_CI') != 'true',
reason='The unittest need to download image, so only run on CI',
)
def test_agnostic_sandbox_jupyter_agentskills_fileop_pwd(temp_dir):
for base_sandbox_image in ['ubuntu:22.04', 'debian:11']:
config = AppConfig(
sandbox=SandboxConfig(
box_type='ssh',
container_image=base_sandbox_image,
enable_auto_lint=False,
),
persist_sandbox=False,
)
assert not config.sandbox.enable_auto_lint
box = create_docker_box_from_app_config(temp_dir, config)
_test_sandbox_jupyter_agentskills_fileop_pwd_impl(box, config)
def test_sandbox_jupyter_plugin_backticks(temp_dir):
config = AppConfig(
sandbox=SandboxConfig(
box_type='ssh',
),
persist_sandbox=False,
)
box = DockerSSHBox(
config=config.sandbox,
persist_sandbox=config.persist_sandbox,
workspace_mount_path=temp_dir,
sandbox_workspace_dir=config.workspace_mount_path_in_sandbox,
cache_dir=config.cache_dir,
run_as_devin=True,
ssh_hostname=config.ssh_hostname,
ssh_password=config.ssh_password,
ssh_port=config.ssh_port,
)
box.init_plugins([JupyterRequirement])
test_code = "print('Hello, `World`!')"
expected_write_command = (
"cat > /tmp/opendevin_jupyter_temp.py <<'EOL'\n" f'{test_code}\n' 'EOL'
)
expected_execute_command = 'cat /tmp/opendevin_jupyter_temp.py | execute_cli'
exit_code, output = box.execute(expected_write_command)
exit_code, output = box.execute(expected_execute_command)
print(output)
assert exit_code == 0, 'The exit code should be 0 for ' + box.__class__.__name__
assert output.strip() == 'Hello, `World`!', (
'The output should be the same as the input for ' + box.__class__.__name__
)
box.close()