Skip to content

Commit bb7ddf2

Browse files
rushtehranirushtehrani
authored
Merge pull request #14 from onepanelio/dev
chore: dev > master (v0.12.0)
2 parents 3317104 + b27b27a commit bb7ddf2

File tree

8 files changed

+662
-391
lines changed

8 files changed

+662
-391
lines changed

‎examples/basic_streaming.py‎

Lines changed: 0 additions & 9 deletions
This file was deleted.

‎examples/execute-workflow.ipynb‎

Lines changed: 251 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,251 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": 1,
6+
"metadata": {},
7+
"outputs": [
8+
{
9+
"name": "stdout",
10+
"output_type": "stream",
11+
"text": [
12+
"Requirement already satisfied: onepanel-sdk in /opt/conda/lib/python3.6/site-packages (0.12.0)\n",
13+
"Requirement already satisfied: python-dateutil in /opt/conda/lib/python3.6/site-packages (from onepanel-sdk) (2.8.1)\n",
14+
"Requirement already satisfied: certifi in /opt/conda/lib/python3.6/site-packages (from onepanel-sdk) (2020.6.20)\n",
15+
"Requirement already satisfied: six>=1.10 in /opt/conda/lib/python3.6/site-packages (from onepanel-sdk) (1.15.0)\n",
16+
"Requirement already satisfied: urllib3>=1.15 in /opt/conda/lib/python3.6/site-packages (from onepanel-sdk) (1.25.10)\n"
17+
]
18+
}
19+
],
20+
"source": [
21+
"!pip install onepanel-sdk"
22+
]
23+
},
24+
{
25+
"cell_type": "code",
26+
"execution_count": 2,
27+
"metadata": {},
28+
"outputs": [],
29+
"source": [
30+
"import os\n",
31+
"import sys\n",
32+
"from time import sleep\n",
33+
"from pprint import pprint\n",
34+
"\n",
35+
"import onepanel.core.api\n",
36+
"from onepanel.core.api.rest import ApiException\n",
37+
"from onepanel.core.api.models import Parameter"
38+
]
39+
},
40+
{
41+
"cell_type": "code",
42+
"execution_count": 4,
43+
"metadata": {},
44+
"outputs": [
45+
{
46+
"name": "stdout",
47+
"output_type": "stream",
48+
"text": [
49+
"{'created_at': '2020-08-26T22:41:51Z',\n",
50+
" 'finished_at': None,\n",
51+
" 'labels': None,\n",
52+
" 'manifest': None,\n",
53+
" 'metadata': {'url': 'http://app.rush-gcs.onepanel.site/rush/workflows/pytorch-training-qbc6z'},\n",
54+
" 'name': 'pytorch-training-qbc6z',\n",
55+
" 'parameters': None,\n",
56+
" 'phase': None,\n",
57+
" 'started_at': None,\n",
58+
" 'uid': 'pytorch-training-qbc6z',\n",
59+
" 'workflow_template': {'created_at': '2020-08-26T05:25:08Z',\n",
60+
" 'cron_stats': None,\n",
61+
" 'is_archived': None,\n",
62+
" 'is_latest': None,\n",
63+
" 'labels': None,\n",
64+
" 'manifest': 'entrypoint: main\\n'\n",
65+
" 'arguments:\\n'\n",
66+
" ' parameters:\\n'\n",
67+
" ' - name: source\\n'\n",
68+
" ' value: '\n",
69+
" 'https://github.com/onepanelio/pytorch-examples.git\\n'\n",
70+
" ' - name: command\\n'\n",
71+
" ' value: \"python mnist/main.py '\n",
72+
" '--epochs=1\"\\n'\n",
73+
" 'volumeClaimTemplates:\\n'\n",
74+
" ' - metadata:\\n'\n",
75+
" ' name: data\\n'\n",
76+
" ' spec:\\n'\n",
77+
" ' accessModes: [ \"ReadWriteOnce\" ]\\n'\n",
78+
" ' resources:\\n'\n",
79+
" ' requests:\\n'\n",
80+
" ' storage: 2Gi\\n'\n",
81+
" ' - metadata:\\n'\n",
82+
" ' name: output\\n'\n",
83+
" ' spec:\\n'\n",
84+
" ' accessModes: [ \"ReadWriteOnce\" ]\\n'\n",
85+
" ' resources:\\n'\n",
86+
" ' requests:\\n'\n",
87+
" ' storage: 2Gi\\n'\n",
88+
" 'templates:\\n'\n",
89+
" ' - name: main\\n'\n",
90+
" ' dag:\\n'\n",
91+
" ' tasks:\\n'\n",
92+
" ' - name: train-model\\n'\n",
93+
" ' template: pytorch\\n'\n",
94+
" '# Uncomment section below to send metrics '\n",
95+
" 'to Slack\\n'\n",
96+
" '# - name: notify-in-slack\\n'\n",
97+
" '# dependencies: [train-model]\\n'\n",
98+
" '# template: slack-notify-success\\n'\n",
99+
" '# arguments:\\n'\n",
100+
" '# parameters:\\n'\n",
101+
" '# - name: status\\n'\n",
102+
" '# value: '\n",
103+
" '\"{{tasks.train-model.status}}\"\\n'\n",
104+
" '# artifacts:\\n'\n",
105+
" '# - name: metrics\\n'\n",
106+
" '# from: '\n",
107+
" '\"{{tasks.train-model.outputs.artifacts.sys-metrics}}\"\\n'\n",
108+
" ' - name: pytorch\\n'\n",
109+
" ' inputs:\\n'\n",
110+
" ' artifacts:\\n'\n",
111+
" ' - name: src\\n'\n",
112+
" ' path: /mnt/src\\n'\n",
113+
" ' git:\\n'\n",
114+
" ' repo: '\n",
115+
" '\"{{workflow.parameters.source}}\"\\n'\n",
116+
" ' outputs:\\n'\n",
117+
" ' artifacts:\\n'\n",
118+
" ' - name: model\\n'\n",
119+
" ' path: /mnt/output\\n'\n",
120+
" ' optional: true\\n'\n",
121+
" ' archive:\\n'\n",
122+
" ' none: {}\\n'\n",
123+
" ' container:\\n'\n",
124+
" ' image: pytorch/pytorch:latest\\n'\n",
125+
" ' command: [sh,-c]\\n'\n",
126+
" ' args: '\n",
127+
" '[\"{{workflow.parameters.command}}\"]\\n'\n",
128+
" ' workingDir: /mnt/src\\n'\n",
129+
" ' volumeMounts:\\n'\n",
130+
" ' - name: data\\n'\n",
131+
" ' mountPath: /mnt/data\\n'\n",
132+
" ' - name: output\\n'\n",
133+
" ' mountPath: /mnt/output\\n'\n",
134+
" ' - name: slack-notify-success\\n'\n",
135+
" ' container:\\n'\n",
136+
" ' image: technosophos/slack-notify\\n'\n",
137+
" ' command: [sh,-c]\\n'\n",
138+
" \" args: ['SLACK_USERNAME=Worker \"\n",
139+
" 'SLACK_TITLE=\"{{workflow.name}} '\n",
140+
" '{{inputs.parameters.status}}\" '\n",
141+
" 'SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd '\n",
142+
" 'SLACK_MESSAGE=$(cat /tmp/metrics.json)} '\n",
143+
" \"./slack-notify']\\n\"\n",
144+
" ' inputs:\\n'\n",
145+
" ' parameters:\\n'\n",
146+
" ' - name: status\\n'\n",
147+
" ' artifacts:\\n'\n",
148+
" ' - name: metrics\\n'\n",
149+
" ' path: /tmp/metrics.json\\n'\n",
150+
" ' optional: true\\n',\n",
151+
" 'modified_at': '2020-08-26T05:25:08Z',\n",
152+
" 'name': 'PyTorch Training',\n",
153+
" 'parameters': [{'display_name': None,\n",
154+
" 'hint': None,\n",
155+
" 'name': 'source',\n",
156+
" 'options': None,\n",
157+
" 'required': None,\n",
158+
" 'type': None,\n",
159+
" 'value': 'https://github.com/onepanelio/pytorch-examples.git',\n",
160+
" 'visibility': 'public'},\n",
161+
" {'display_name': None,\n",
162+
" 'hint': None,\n",
163+
" 'name': 'command',\n",
164+
" 'options': None,\n",
165+
" 'required': None,\n",
166+
" 'type': None,\n",
167+
" 'value': 'python mnist/main.py '\n",
168+
" '--epochs=1',\n",
169+
" 'visibility': 'public'}],\n",
170+
" 'stats': {'completed': None,\n",
171+
" 'failed': None,\n",
172+
" 'last_executed': '0001-01-01T00:00:00Z',\n",
173+
" 'running': None,\n",
174+
" 'terminated': None,\n",
175+
" 'total': None},\n",
176+
" 'uid': 'pytorch-training',\n",
177+
" 'version': '1598419508627499422',\n",
178+
" 'versions': None}}\n"
179+
]
180+
}
181+
],
182+
"source": [
183+
"# If outside of Onepanel\n",
184+
"# token = '<auth-token>'\n",
185+
"# Set environment variables for:\n",
186+
"# ONEPANEL_API_URL, example: https://app.domain.com/api\n",
187+
"# ONEPANEL_RESOURCE_NAMESPACE, example: default\n",
188+
"\n",
189+
"# If inside of Onepanel, get mounted service account token to use as API Key\n",
190+
"with open('/var/run/secrets/kubernetes.io/serviceaccount/token') as f:\n",
191+
" token = f.read()\n",
192+
"\n",
193+
"# Configure API key authorization: Bearer\n",
194+
"configuration = onepanel.core.api.Configuration(\n",
195+
" host=os.getenv('ONEPANEL_API_URL'),\n",
196+
" api_key={\n",
197+
" 'Bearer': token\n",
198+
" }\n",
199+
")\n",
200+
"configuration.api_key_prefix['Bearer'] = 'Bearer'\n",
201+
"\n",
202+
"# Enter a context with an instance of the API client\n",
203+
"with onepanel.core.api.ApiClient(configuration) as api_client:\n",
204+
" # Create an instance of the API class\n",
205+
" api_instance = onepanel.core.api.WorkflowServiceApi(api_client)\n",
206+
"\n",
207+
" namespace = os.getenv('ONEPANEL_RESOURCE_NAMESPACE')\n",
208+
" params = []\n",
209+
" params.append(Parameter(name='source', value='https://github.com/onepanelio/pytorch-examples.git'))\n",
210+
" params.append(Parameter(name='command', value='python mnist/main.py --epochs=1'))\n",
211+
" body = onepanel.core.api.CreateWorkflowExecutionBody(\n",
212+
" parameters=params,\n",
213+
" workflow_template_uid='pytorch-training',\n",
214+
" )\n",
215+
" try:\n",
216+
" workflow = api_instance.create_workflow_execution(namespace, body)\n",
217+
" pprint(workflow)\n",
218+
" except ApiException as e:\n",
219+
" print('Exception when calling WorkflowServiceApi->create_workflow_execution: %s\\n' % e)"
220+
]
221+
},
222+
{
223+
"cell_type": "code",
224+
"execution_count": null,
225+
"metadata": {},
226+
"outputs": [],
227+
"source": []
228+
}
229+
],
230+
"metadata": {
231+
"kernelspec": {
232+
"display_name": "Python 3",
233+
"language": "python",
234+
"name": "python3"
235+
},
236+
"language_info": {
237+
"codemirror_mode": {
238+
"name": "ipython",
239+
"version": 3
240+
},
241+
"file_extension": ".py",
242+
"mimetype": "text/x-python",
243+
"name": "python",
244+
"nbconvert_exporter": "python",
245+
"pygments_lexer": "ipython3",
246+
"version": "3.6.11"
247+
}
248+
},
249+
"nbformat": 4,
250+
"nbformat_minor": 4
251+
}

‎examples/launch-workspace.ipynb‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": 3,
6+
"metadata": {},
7+
"outputs": [
8+
{
9+
"name": "stdout",
10+
"output_type": "stream",
11+
"text": [
12+
"Requirement already satisfied: onepanel-sdk in /opt/conda/lib/python3.6/site-packages (0.12.0)\n",
13+
"Requirement already satisfied: urllib3>=1.15 in /opt/conda/lib/python3.6/site-packages (from onepanel-sdk) (1.25.10)\n",
14+
"Requirement already satisfied: certifi in /opt/conda/lib/python3.6/site-packages (from onepanel-sdk) (2020.6.20)\n",
15+
"Requirement already satisfied: six>=1.10 in /opt/conda/lib/python3.6/site-packages (from onepanel-sdk) (1.15.0)\n",
16+
"Requirement already satisfied: python-dateutil in /opt/conda/lib/python3.6/site-packages (from onepanel-sdk) (2.8.1)\n"
17+
]
18+
}
19+
],
20+
"source": [
21+
"!pip install onepanel-sdk"
22+
]
23+
},
24+
{
25+
"cell_type": "code",
26+
"execution_count": 1,
27+
"metadata": {},
28+
"outputs": [],
29+
"source": [
30+
"import os\n",
31+
"import sys\n",
32+
"from time import sleep\n",
33+
"\n",
34+
"import onepanel.core.api\n",
35+
"from onepanel.core.api.rest import ApiException\n",
36+
"from onepanel.core.api.models import Parameter"
37+
]
38+
},
39+
{
40+
"cell_type": "code",
41+
"execution_count": 8,
42+
"metadata": {},
43+
"outputs": [
44+
{
45+
"name": "stdout",
46+
"output_type": "stream",
47+
"text": [
48+
"Waiting for workspace to launch...\n",
49+
"\n",
50+
"Workspace is running at http://jupyterlab-workspace--rush.rush-gcs.onepanel.site.\n",
51+
"\n"
52+
]
53+
}
54+
],
55+
"source": [
56+
"# If outside of Onepanel\n",
57+
"# token = '<auth-token>'\n",
58+
"# Set environment variables for:\n",
59+
"# ONEPANEL_API_URL, example: https://app.domain.com/api\n",
60+
"# ONEPANEL_RESOURCE_NAMESPACE, example: default\n",
61+
"\n",
62+
"# If inside of Onepanel, get mounted service account token to use as API Key\n",
63+
"with open('/var/run/secrets/kubernetes.io/serviceaccount/token') as f:\n",
64+
" token = f.read()\n",
65+
"\n",
66+
"# Configure API key authorization: Bearer\n",
67+
"configuration = onepanel.core.api.Configuration(\n",
68+
" host=os.getenv('ONEPANEL_API_URL'),\n",
69+
" api_key={\n",
70+
" 'Bearer': token\n",
71+
" }\n",
72+
")\n",
73+
"configuration.api_key_prefix['Bearer'] = 'Bearer'\n",
74+
"\n",
75+
"# Enter a context with an instance of the API client\n",
76+
"with onepanel.core.api.ApiClient(configuration) as api_client:\n",
77+
" # Create an instance of the API class\n",
78+
" api_instance = onepanel.core.api.WorkspaceServiceApi(api_client)\n",
79+
"\n",
80+
" namespace = os.getenv('ONEPANEL_RESOURCE_NAMESPACE')\n",
81+
" params = []\n",
82+
" params.append(Parameter(name='sys-name', value='JupyterLab Workspace'))\n",
83+
" params.append(Parameter(name='sys-node-pool', value='Standard_NC6'))\n",
84+
" body = onepanel.core.api.CreateWorkspaceBody(\n",
85+
" parameters=params,\n",
86+
" workspace_template_uid='jupyterlab',\n",
87+
" )\n",
88+
" try:\n",
89+
" workspace = api_instance.create_workspace(namespace, body)\n",
90+
" uid = workspace.uid\n",
91+
" print('Waiting for workspace to launch...\\n')\n",
92+
" while workspace.status.phase != 'Running':\n",
93+
" workspace = api_instance.get_workspace(namespace, uid)\n",
94+
" sleep(15)\n",
95+
" print('Workspace is running at %s.\\n' % workspace.url)\n",
96+
" except ApiException as e:\n",
97+
" print(\"Exception when calling WorkspaceServiceApi->create_workspace: %s\\n\" % e)"
98+
]
99+
},
100+
{
101+
"cell_type": "code",
102+
"execution_count": null,
103+
"metadata": {},
104+
"outputs": [],
105+
"source": []
106+
}
107+
],
108+
"metadata": {
109+
"kernelspec": {
110+
"display_name": "Python 3",
111+
"language": "python",
112+
"name": "python3"
113+
},
114+
"language_info": {
115+
"codemirror_mode": {
116+
"name": "ipython",
117+
"version": 3
118+
},
119+
"file_extension": ".py",
120+
"mimetype": "text/x-python",
121+
"name": "python",
122+
"nbconvert_exporter": "python",
123+
"pygments_lexer": "ipython3",
124+
"version": "3.6.11"
125+
}
126+
},
127+
"nbformat": 4,
128+
"nbformat_minor": 4
129+
}

0 commit comments

Comments
 (0)