Skip to content

Commit dcb5d1c

Browse files
rbrenrbren
authored
Add permanent storage option for EventStream (OpenHands#1697)
* add storage classes * add minio * add event stream storage * storage test working * use fixture * event stream test passing * better serialization * factor out serialization pkg * move more serialization * fix tests * fix test * remove __all__ * add rehydration test * add more rehydration test * fix fixture * fix dict init * update tests * lock * regenerate tests * Update opendevin/events/stream.py * revert tests * revert old integration tests * only add fields if present * regen tests * pin pyarrow * fix unit tests * remove cause from memories * revert tests * regen tests
1 parent beb74a1 commit dcb5d1c

73 files changed

Lines changed: 1022 additions & 809 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,3 +204,4 @@ cache
204204
config.toml
205205

206206
test_results*
207+
/_test_files_tmp/

‎agenthub/SWE_agent/agent.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
MessageAction,
88
)
99
from opendevin.events.observation import Observation
10+
from opendevin.events.serialization.event import event_to_memory
1011
from opendevin.llm.llm import LLM
1112

1213
from .parser import parse_command
@@ -36,7 +37,7 @@ def __init__(self, llm: LLM):
3637

3738
def _remember(self, action: Action, observation: Observation) -> None:
3839
"""Agent has a limited memory of the few steps implemented as a queue"""
39-
memory = MEMORY_FORMAT(action.to_memory(), observation.to_memory())
40+
memory = MEMORY_FORMAT(event_to_memory(action), event_to_memory(observation))
4041
self.running_memory.append(memory)
4142

4243
def _think_act(self, messages: list[dict]) -> tuple[Action, str]:

‎agenthub/dummy_agent/agent.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
NullObservation,
2525
Observation,
2626
)
27+
from opendevin.events.serialization.event import event_to_dict
2728
from opendevin.llm.llm import LLM
2829

2930
"""
@@ -138,8 +139,8 @@ def step(self, state: State) -> Action:
138139
expected_observations = prev_step['observations']
139140
hist_start = len(state.history) - len(expected_observations)
140141
for i in range(len(expected_observations)):
141-
hist_obs = state.history[hist_start + i][1].to_dict()
142-
expected_obs = expected_observations[i].to_dict()
142+
hist_obs = event_to_dict(state.history[hist_start + i][1])
143+
expected_obs = event_to_dict(expected_observations[i])
143144
if (
144145
'command_id' in hist_obs['extras']
145146
and hist_obs['extras']['command_id'] != -1

‎agenthub/micro/agent.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@
33
from opendevin.controller.agent import Agent
44
from opendevin.controller.state.state import State
55
from opendevin.core.utils import json
6-
from opendevin.events.action import Action, action_from_dict
6+
from opendevin.events.action import Action
7+
from opendevin.events.serialization.action import action_from_dict
78
from opendevin.llm.llm import LLM
89

910
from .instructions import instructions

‎agenthub/monologue_agent/agent.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
NullObservation,
2323
Observation,
2424
)
25+
from opendevin.events.serialization.event import event_to_memory
2526
from opendevin.llm.llm import LLM
2627
from opendevin.memory.condenser import MemoryCondenser
2728
from opendevin.memory.history import ShortTermHistory
@@ -186,7 +187,7 @@ def _add_initial_thoughts(self, task):
186187
observation = BrowserOutputObservation(
187188
content=thought, url='', screenshot=''
188189
)
189-
self._add_event(observation.to_memory())
190+
self._add_event(event_to_memory(observation))
190191
previous_action = ''
191192
else:
192193
action: Action = NullAction()
@@ -213,7 +214,7 @@ def _add_initial_thoughts(self, task):
213214
previous_action = ActionType.BROWSE
214215
else:
215216
action = MessageAction(thought)
216-
self._add_event(action.to_memory())
217+
self._add_event(event_to_memory(action))
217218

218219
def step(self, state: State) -> Action:
219220
"""
@@ -229,8 +230,8 @@ def step(self, state: State) -> Action:
229230
goal = state.get_current_user_intent()
230231
self._initialize(goal)
231232
for prev_action, obs in state.updated_info:
232-
self._add_event(prev_action.to_memory())
233-
self._add_event(obs.to_memory())
233+
self._add_event(event_to_memory(prev_action))
234+
self._add_event(event_to_memory(obs))
234235

235236
state.updated_info = []
236237

‎agenthub/monologue_agent/utils/prompts.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@
22
from opendevin.core.utils import json
33
from opendevin.events.action import (
44
Action,
5-
action_from_dict,
65
)
76
from opendevin.events.observation import (
87
CmdOutputObservation,
98
)
9+
from opendevin.events.serialization.action import action_from_dict
1010

1111
ACTION_PROMPT = """
1212
You're a thoughtful robot. Your main task is this:

‎agenthub/planner_agent/prompt.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,12 @@
55
from opendevin.events.action import (
66
Action,
77
NullAction,
8-
action_from_dict,
98
)
109
from opendevin.events.observation import (
1110
NullObservation,
1211
)
12+
from opendevin.events.serialization.action import action_from_dict
13+
from opendevin.events.serialization.event import event_to_memory
1314

1415
HISTORY_SIZE = 10
1516

@@ -139,10 +140,10 @@ def get_prompt(state: State) -> str:
139140
latest_action: Action = NullAction()
140141
for action, observation in sub_history:
141142
if not isinstance(action, NullAction):
142-
history_dicts.append(action.to_memory())
143+
history_dicts.append(event_to_memory(action))
143144
latest_action = action
144145
if not isinstance(observation, NullObservation):
145-
observation_dict = observation.to_memory()
146+
observation_dict = event_to_memory(observation)
146147
history_dicts.append(observation_dict)
147148
history_str = json.dumps(history_dicts, indent=2)
148149
current_task = state.root_task.get_current_task()
@@ -152,7 +153,7 @@ def get_prompt(state: State) -> str:
152153
plan_status += "\nIf it's not achievable AND verifiable with a SINGLE action, you MUST break it down into subtasks NOW."
153154
else:
154155
plan_status = "You're not currently working on any tasks. Your next action MUST be to mark a task as in_progress."
155-
hint = get_hint(latest_action.to_dict()['action'])
156+
hint = get_hint(event_to_memory(latest_action).get('action', ''))
156157
logger.info('HINT:\n' + hint, extra={'msg_type': 'INFO'})
157158
task = state.get_current_user_intent()
158159
return prompt % {

‎opendevin/core/config.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,8 @@ class AppConfig(metaclass=Singleton):
7171
llm: LLMConfig = field(default_factory=LLMConfig)
7272
agent: AgentConfig = field(default_factory=AgentConfig)
7373
runtime: str = 'server'
74+
file_store: str = 'memory'
75+
file_store_path: str = '/tmp/file_store'
7476
workspace_base: str = os.getcwd()
7577
workspace_mount_path: str = os.getcwd()
7678
workspace_mount_path_in_sandbox: str = '/workspace'

‎opendevin/core/main.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ async def main(task_str: str = '', exit_on_message: bool = False) -> AgentState:
7676
AgentCls: Type[Agent] = Agent.get_cls(args.agent_cls)
7777
agent = AgentCls(llm=llm)
7878

79-
event_stream = EventStream()
79+
event_stream = EventStream('main')
8080
controller = AgentController(
8181
agent=agent,
8282
max_iterations=args.max_iterations,

‎opendevin/events/action/__init__.py‎

Lines changed: 0 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
from opendevin.core.exceptions import AgentMalformedActionError
2-
31
from .action import Action
42
from .agent import (
53
AgentDelegateAction,
@@ -16,49 +14,6 @@
1614
from .message import MessageAction
1715
from .tasks import AddTaskAction, ModifyTaskAction
1816

19-
actions = (
20-
CmdKillAction,
21-
CmdRunAction,
22-
IPythonRunCellAction,
23-
BrowseURLAction,
24-
FileReadAction,
25-
FileWriteAction,
26-
AgentRecallAction,
27-
AgentFinishAction,
28-
AgentRejectAction,
29-
AgentDelegateAction,
30-
AddTaskAction,
31-
ModifyTaskAction,
32-
ChangeAgentStateAction,
33-
MessageAction,
34-
)
35-
36-
ACTION_TYPE_TO_CLASS = {action_class.action: action_class for action_class in actions} # type: ignore[attr-defined]
37-
38-
39-
def action_from_dict(action: dict) -> Action:
40-
if not isinstance(action, dict):
41-
raise AgentMalformedActionError('action must be a dictionary')
42-
action = action.copy()
43-
if 'action' not in action:
44-
raise AgentMalformedActionError(f"'action' key is not found in {action=}")
45-
if not isinstance(action['action'], str):
46-
raise AgentMalformedActionError(
47-
f"'{action['action']=}' is not defined. Available actions: {ACTION_TYPE_TO_CLASS.keys()}"
48-
)
49-
action_class = ACTION_TYPE_TO_CLASS.get(action['action'])
50-
if action_class is None:
51-
raise AgentMalformedActionError(
52-
f"'{action['action']=}' is not defined. Available actions: {ACTION_TYPE_TO_CLASS.keys()}"
53-
)
54-
args = action.get('args', {})
55-
try:
56-
decoded_action = action_class(**args)
57-
except TypeError:
58-
raise AgentMalformedActionError(f'action={action} has the wrong arguments')
59-
return decoded_action
60-
61-
6217
__all__ = [
6318
'Action',
6419
'NullAction',

0 commit comments

Comments
 (0)