-
Notifications
You must be signed in to change notification settings - Fork 29
Expand file tree
/
Copy pathdssclient.py
More file actions
2443 lines (1969 loc) · 109 KB
/
dssclient.py
File metadata and controls
2443 lines (1969 loc) · 109 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
import json, warnings
import sys
if sys.version_info >= (3,0):
import urllib.parse
dku_quote_fn = urllib.parse.quote
else:
import urllib
dku_quote_fn = urllib.quote
from requests import Session
from requests import exceptions
from requests.auth import HTTPBasicAuth
from .iam.settings import DSSSSOSettings, DSSLDAPSettings, DSSAzureADSettings
from .dss.data_collection import DSSDataCollection, DSSDataCollectionListItem
from .dss.data_directories_footprint import DSSDataDirectoriesFootprint
from .dss.feature_store import DSSFeatureStore
from .dss.notebook import DSSNotebook
from .dss.future import DSSFuture
from .dss.projectfolder import DSSProjectFolder
from .dss.project import DSSProject
from .dss.app import DSSApp, DSSAppListItem
from .dss.businessapp import DSSBusinessApp, DSSBusinessAppListItem
from .dss.plugin import DSSPlugin
from .dss.admin import DSSGlobalApiKeyListItem, DSSPersonalApiKeyListItem, DSSUser, DSSUserActivity, DSSOwnUser, DSSGroup, DSSUserInfo, DSSGroupInfo, DSSConnection, DSSConnectionListItem, DSSGeneralSettings, DSSCodeEnv, DSSGlobalApiKey, DSSCluster, DSSCodeStudioTemplate, DSSCodeStudioTemplateListItem, DSSGlobalUsageSummary, DSSInstanceVariables, DSSPersonalApiKey, DSSAuthorizationMatrix, DSSLLMCostLimitingCounters
from .dss.messaging_channel import DSSMailMessagingChannel, DSSMessagingChannelListItem, DSSMessagingChannel, SMTPMessagingChannelCreator, AWSSESMailMessagingChannelCreator, MicrosoftGraphMailMessagingChannelCreator, SlackMessagingChannelCreator, MSTeamsMessagingChannelCreator, GoogleChatMessagingChannelCreator, TwilioMessagingChannelCreator, ShellMessagingChannelCreator
from .dss.meaning import DSSMeaning
from .dss.sqlquery import DSSSQLQuery
from .dss.discussion import DSSObjectDiscussions
from .dss.apideployer import DSSAPIDeployer
from .dss.projectdeployer import DSSProjectDeployer
from .dss.project_standards import DSSProjectStandards
from .dss.unifiedmonitoring import DSSUnifiedMonitoring
from .dss.utils import DSSInfoMessages, Enum
from .dss.workspace import DSSWorkspace
from .dss.enterprise_asset_library import DSSEnterpriseAssetLibrary
import os.path as osp
from .utils import dku_basestring_type, handle_http_exception
from .govern_client import GovernClient
class DSSClient(object):
"""Entry point for the DSS API client"""
def __init__(self, host, api_key=None, internal_ticket=None, extra_headers=None, no_check_certificate=False, client_certificate=None, **kwargs):
"""Initialize a new DSS API client.
Args:
host (str): The host URL of the DSS instance (e.g., "http://localhost:11200")
api_key (str, optional): API key for authentication. Can be managed in DSS project page or global settings.
internal_ticket (str, optional): Internal ticket for authentication.
extra_headers (dict, optional): Additional HTTP headers to include in requests.
no_check_certificate (bool, optional): If True, disables SSL certificate verification.
Defaults to False.
client_certificate (str or tuple, optional): Path to client certificate file or tuple of (cert, key) paths.
**kwargs: Additional keyword arguments. Note: 'insecure_tls' is deprecated in favor of no_check_certificate.
Note:
The API key determines which operations are allowed for the client.
If no_check_certificate is True, SSL certificate verification will be disabled.
If client_certificate is provided, it will be used for client certificate authentication.
"""
if "insecure_tls" in kwargs:
# Backward compatibility before removing insecure_tls option
warnings.warn("insecure_tls field is now deprecated. It has been replaced by no_check_certificate.", DeprecationWarning)
no_check_certificate = kwargs.get("insecure_tls") or no_check_certificate
self.api_key = api_key
self.internal_ticket = internal_ticket
self.host = host
self._session = Session()
if no_check_certificate:
self._session.verify = False
if client_certificate:
self._session.cert = client_certificate
if self.api_key is not None:
self._session.auth = HTTPBasicAuth(self.api_key, "")
elif self.internal_ticket is not None:
self._session.headers.update({"X-DKU-APITicket" : self.internal_ticket})
else:
raise ValueError("API Key is required")
if extra_headers is not None:
self._session.headers.update(extra_headers)
########################################################
# Futures
########################################################
def list_futures(self, as_objects=False, all_users=False):
"""
List the currently-running long tasks (a.k.a futures)
:param boolean as_objects: if True, each returned item will be a :class:`dataikuapi.dss.future.DSSFuture`
:param boolean all_users: if True, returns futures for all users (requires admin privileges). Else, only returns futures for the user associated with the current authentication context (if any)
:return: list of futures. if as_objects is True, each future in the list is a :class:`dataikuapi.dss.future.DSSFuture`. Else, each future in the list is a dict. Each dict contains at least a 'jobId' field
:rtype: list of :class:`dataikuapi.dss.future.DSSFuture` or list of dict
"""
list = self._perform_json("GET", "/futures/", params={"withScenarios":False, "withNotScenarios":True, 'allUsers' : all_users})
if as_objects:
return [DSSFuture(self, state['jobId'], state) for state in list]
else:
return list
def list_running_scenarios(self, all_users=False):
"""
List the running scenarios
:param boolean all_users: if True, returns scenarios for all users (requires admin privileges). Else, only returns scenarios for the user associated with the current authentication context (if any)
:return: list of running scenarios, each one as a dict containing at least a "jobId" field for the
future hosting the scenario run, and a "payload" field with scenario identifiers
:rtype: list of dicts
"""
return self._perform_json("GET", "/futures/", params={"withScenarios":True, "withNotScenarios":False, 'allUsers' : all_users})
def get_future(self, job_id):
"""
Get a handle to interact with a specific long task (a.k.a future). This notably allows aborting this future.
:param str job_id: the identifier of the desired future (which can be returned by :py:meth:`list_futures` or :py:meth:`list_running_scenarios`)
:returns: A handle to interact the future
:rtype: :class:`dataikuapi.dss.future.DSSFuture`
"""
return DSSFuture(self, job_id)
########################################################
# Notebooks
########################################################
def list_running_notebooks(self, as_objects=True):
"""
List the currently-running Jupyter notebooks
:param boolean as_objects: if True, each returned item will be a :class:`dataikuapi.dss.notebook.DSSNotebook`
:return: list of notebooks. if as_objects is True, each entry in the list is a :class:`dataikuapi.dss.notebook.DSSNotebook`. Else, each item in the list is a dict which contains at least a "name" field.
:rtype: list of :class:`dataikuapi.dss.notebook.DSSNotebook` or list of dict
"""
notebook_list = self._perform_json("GET", "/admin/notebooks/")
if as_objects:
return [DSSNotebook(self, notebook['projectKey'], notebook['name'], notebook) for notebook in notebook_list]
else:
return notebook_list
########################################################
# Project folders
########################################################
def get_root_project_folder(self):
"""
Get a handle to interact with the root project folder.
:returns: A :class:`dataikuapi.dss.projectfolder.DSSProjectFolder` to interact with this project folder
"""
return self.get_project_folder("ROOT")
def get_project_folder(self, project_folder_id):
"""
Get a handle to interact with a project folder.
:param str project_folder_id: the project folder ID of the desired project folder
:returns: A :class:`dataikuapi.dss.projectfolder.DSSProjectFolder` to interact with this project folder
"""
data = self._perform_json("GET", "/project-folders/%s" % project_folder_id)
return DSSProjectFolder(self, data)
########################################################
# Projects
########################################################
def list_project_keys(self):
"""
List the project keys (=project identifiers).
:returns: list of project keys identifiers, as strings
:rtype: list of strings
"""
return [x["projectKey"] for x in self._perform_json("GET", "/projects/")]
def list_projects(self, include_location=False):
"""
List the projects
:param bool include_location: whether to include project locations (slower)
:returns: a list of projects, each as a dict. Each dict contains at least a 'projectKey' field
:rtype: list of dicts
"""
return self._perform_json("GET", "/projects/", params={"includeLocation": include_location})
def get_project(self, project_key):
"""
Get a handle to interact with a specific project.
:param str project_key: the project key of the desired project
:returns: A :class:`dataikuapi.dss.project.DSSProject` to interact with this project
"""
return DSSProject(self, project_key)
def get_default_project(self):
"""
Get a handle to the current default project, if available (i.e. if dataiku.default_project_key() is valid)
"""
import dataiku
return DSSProject(self, dataiku.default_project_key())
def create_project(self, project_key, name, owner, description=None, settings=None, project_folder_id=None, permissions=None, tags=[]):
"""
Creates a new project, and return a project handle to interact with it.
Note: this call requires an API key with admin rights or the rights to create a project
:param str project_key: the identifier to use for the project. Must be globally unique
:param str name: the display name for the project.
:param str owner: the login of the owner of the project.
:param str description: a description for the project.
:param dict settings: Initial settings for the project (can be modified later). The exact possible settings are not documented.
:param str project_folder_id: the project folder ID in which the project will be created (root project folder if not specified)
:param list[dict] permissions: Initial permissions for the project (can be modified later). Each dict contains a 'group' and permissions given to that group.
:param list[str] tags: a list of tags for the project
:returns: A :class:`dataikuapi.dss.project.DSSProject` project handle to interact with this project
"""
params = {}
if project_folder_id is not None:
params["projectFolderId"] = project_folder_id
if permissions is None:
permissions = []
resp = self._perform_text(
"POST", "/projects/", body={
"projectKey" : project_key,
"name" : name,
"owner" : owner,
"settings" : settings,
"description" : description,
"permissions" : permissions,
"tags": tags
}, params=params)
return DSSProject(self, project_key)
########################################################
# Apps
########################################################
def list_apps(self, as_type="listitems"):
"""
List the apps.
:param str as_type: How to return the list. Supported values are "listitems" and "objects" (defaults to **listitems**).
:returns: The list of the apps. If "as_type" is "listitems", each one as a :class:`dataikuapi.dss.app.DSSAppListItem`.
If "as_type" is "objects", each one as a :class:`dataikuapi.dss.app.DSSApp`
:rtype: list
"""
items = self._perform_json("GET", "/apps/")
if as_type == "listitems" or as_type == "listitem":
return [DSSAppListItem(self, item) for item in items]
elif as_type == "objects" or as_type == "object":
return [DSSApp(self, item["appId"]) for item in items]
else:
raise ValueError("Unknown as_type")
def get_app(self, app_id):
"""
Get a handle to interact with a specific app.
.. note::
If a project XXXX is an app template, the identifier of the associated app is PROJECT_XXXX
:param str app_id: the id of the desired app
:returns: A :class:`dataikuapi.dss.app.DSSApp` to interact with this project
"""
return DSSApp(self, app_id)
########################################################
# Business Applications
########################################################
def list_business_apps(self, as_type="listitems"):
"""
List the installed Business Applications.
:param str as_type: How to return the list. Supported values are "listitems" and "objects" (defaults to **listitems**).
:returns: The list of the Business Applications. If "as_type" is "listitems", each one as a
:class:`dataikuapi.dss.businessapp.DSSBusinessAppListItem`.
If "as_type" is "objects", each one as a :class:`dataikuapi.dss.businessapp.DSSBusinessApp`
:rtype: list
"""
items = self._perform_json("GET", "/business-apps/")
if as_type == "listitems" or as_type == "listitem":
return [DSSBusinessAppListItem(self, item) for item in items]
elif as_type == "objects" or as_type == "object":
return [DSSBusinessApp(self, item["id"]) for item in items]
else:
raise ValueError("Unknown as_type")
def get_business_app(self, business_app_id):
"""
Get a handle to interact with a specific Business Application.
:param str business_app_id: the id of the desired Business Application
:returns: A :class:`dataikuapi.dss.businessapp.DSSBusinessApp` to interact with this Business Application
:rtype: :class:`dataikuapi.dss.businessapp.DSSBusinessApp`
"""
return DSSBusinessApp(self, business_app_id)
def install_business_app_from_archive(self, fp):
"""
Install or upgrade a Business Application from a zip archive.
Code-env creation must be done separately by calling DSSClient.create_code_env.
.. note::
This call requires an API key with admin rights
:param object fp: A file-like object pointing to a Business Application zip
:return: a future representing the installation/upgrade process
:rtype: :class:`dataikuapi.dss.future.DSSFuture`
"""
files = {'file': fp}
resp = self._perform_json("POST", "/business-apps/install-from-archive", files=files)
return DSSFuture.from_resp(self, resp)
########################################################
# Plugins
########################################################
def list_plugins(self):
"""
List the installed plugins
:returns: list of dict. Each dict contains at least a 'id' field
"""
return self._perform_json("GET", "/plugins/")
def download_plugin_stream(self, plugin_id):
"""
Download a development plugin, as a binary stream
:param str plugin_id: identifier of the plugin to download
:param plugin_id:
:return: the binary stream
"""
return self._perform_raw("GET", "/plugins/%s/download" % plugin_id)
def download_plugin_to_file(self, plugin_id, path):
"""
Download a development plugin to a file
:param str plugin_id: identifier of the plugin to download
:param str path: the path where to download the plugin
:return: None
"""
stream = self.download_plugin_stream(plugin_id)
with open(path, 'wb') as f:
for chunk in stream.iter_content(chunk_size=10000):
if chunk:
f.write(chunk)
f.flush()
def install_plugin_from_archive(self, fp):
"""
Install a plugin from a plugin archive (as a file object)
:param object fp: A file-like object pointing to a plugin archive zip
"""
files = {'file': fp }
self._perform_json("POST", "/plugins/actions/installFromZip", files=files)
def start_install_plugin_from_archive(self, fp):
"""
Install a plugin from a plugin archive (as a file object)
Returns immediately with a future representing the process done asynchronously
:param object fp: A file-like object pointing to a plugin archive zip
:return: A :class:`~dataikuapi.dss.future.DSSFuture` representing the install process
:rtype: :class:`~dataikuapi.dss.future.DSSFuture`
"""
files = {'file': fp }
f = self._perform_json("POST", "/plugins/actions/future/installFromZip", files=files)
return DSSFuture.from_resp(self, f)
def install_plugin_from_store(self, plugin_id):
"""
Install a plugin from the Dataiku plugin store
:param str plugin_id: identifier of the plugin to install
:return: A :class:`~dataikuapi.dss.future.DSSFuture` representing the install process
"""
f = self._perform_json("POST", "/plugins/actions/installFromStore", body={
"pluginId": plugin_id
})
return DSSFuture.from_resp(self, f)
def install_plugin_from_git(self, repository_url, checkout = "master", subpath=None):
"""
Install a plugin from a Git repository. DSS must be setup to allow access to the repository.
:param str repository_url: URL of a Git remote
:param str checkout: branch/tag/SHA1 to commit. For example "master"
:param str subpath: Optional, path within the repository to use as plugin. Should contain a 'plugin.json' file
:return: A :class:`~dataikuapi.dss.future.DSSFuture` representing the install process
"""
f = self._perform_json("POST", "/plugins/actions/installFromGit", body={
"gitRepositoryUrl": repository_url,
"gitCheckout" : checkout,
"gitSubpath": subpath
})
return DSSFuture.from_resp(self, f)
def get_plugin(self, plugin_id):
"""
Get a handle to interact with a specific plugin
:param str plugin_id: the identifier of the desired plugin
:returns: A :class:`dataikuapi.dss.project.DSSPlugin`
"""
return DSSPlugin(self, plugin_id)
########################################################
# SQL queries
########################################################
def sql_query(self, query, connection=None, database=None, dataset_full_name=None, pre_queries=None, post_queries=None, type='sql', extra_conf=None, script_steps=None, script_input_schema=None, script_output_schema=None, script_report_location=None, read_timestamp_without_timezone_as_string=True, read_date_as_string=False, project_key=None, datetimenotz_read_mode="AS_IS", dateonly_read_mode="AS_IS"):
"""
Initiate a SQL, Hive or Impala query and get a handle to retrieve the results of the query.
Internally, the query is run by DSS. The database to run the query on is specified either by
passing a connection name, or by passing a database name, or by passing a dataset full name
(whose connection is then used to retrieve the database)
:param str query: the query to run
:param str connection: the connection on which the query should be run (exclusive of database and dataset_full_name)
:param str database: the database on which the query should be run (exclusive of connection and dataset_full_name)
:param str dataset_full_name: the dataset on the connection of which the query should be run (exclusive of connection and database)
:param list pre_queries: (optional) array of queries to run before the query
:param list post_queries: (optional) array of queries to run after the query
:param str type: the type of query : either 'sql', 'hive' or 'impala' (default: sql)
:param str project_key: The project_key on which the query should be run (especially useful for user isolation/impersonation scenario)
:param str datetimenotz_read_mode: if set to 'AS_IS', read SQL data types that map to the 'datetime no tz' DSS type as such. If set
to 'AS_STRING', read them as strings, straight from the database (ie: conversion to string is
done by the database, according to its own settings). If set to 'AS_DATE', read them as the DSS 'datetime with tz'
type, in the UTC timezone. Default 'AS_IS'
:param str dateonly_read_mode: if set to 'AS_IS', read SQL data types that map to the 'date only' DSS type as such. If set
to 'AS_STRING', read them as strings, straight from the database. If set to 'AS_DATE', read them as the
DSS 'datetime with tz' type, in the UTC timezone. Default 'AS_IS'
:return: a handle on the SQL query
:rtype: :class:`dataikuapi.dss.sqlquery.DSSSQLQuery`
"""
if extra_conf is None:
extra_conf = {}
return DSSSQLQuery(self, query, connection, database, dataset_full_name, pre_queries, post_queries, type, extra_conf, script_steps, script_input_schema, script_output_schema, script_report_location, read_timestamp_without_timezone_as_string, read_date_as_string, datetimenotz_read_mode, dateonly_read_mode, project_key)
########################################################
# Users & Groups (non-admin version)
########################################################
def list_users_info(self):
"""
Gets basic information about users on the DSS instance.
You do not need to be admin to call this
:return: A list of users, as a list of :class:`dataikuapi.dss.admin.DSSUserInfo`
"""
data = self._perform_json("GET", "/users")
return [DSSUserInfo(u) for u in data]
def list_groups_info(self):
"""
Gets basic information about groups on the DSS instance.
You do not need to be admin to call this
:return: A list of groups, as a list of :class:`dataikuapi.dss.admin.DSSGroupInfo`
"""
data = self._perform_json("GET", "/groups")
return [DSSGroupInfo(g) for g in data]
########################################################
# Users
########################################################
def list_users(self, as_objects=False, include_settings=False):
"""
List all users setup on the DSS instance
Note: this call requires an API key with admin rights
:param bool as_objects: Return a list of :class:`dataikuapi.dss.admin.DSSUser` instead of dictionaries. Defaults to False.
:param bool include_settings: Include detailed user settings in the response. Only useful if as_objects is False, as
:class:`dataikuapi.dss.admin.DSSUser` already includes settings by default. Defaults to False.
:return: A list of users, as a list of :class:`dataikuapi.dss.admin.DSSUser` if as_objects is True, else as a list of dicts
:rtype: list of :class:`dataikuapi.dss.admin.DSSUser` or list of dicts
"""
params = {
"includeSettings": include_settings
}
users = self._perform_json("GET", "/admin/users/", params=params)
if as_objects:
return [DSSUser(self, user["login"]) for user in users]
else:
return users
def get_user(self, login):
"""
Get a handle to interact with a specific user
:param str login: the login of the desired user
:return: A :class:`dataikuapi.dss.admin.DSSUser` user handle
"""
return DSSUser(self, login)
def create_user(self, login, password, display_name='', source_type='LOCAL', groups=None, profile='DATA_SCIENTIST', email=None):
"""
Create a user, and return a handle to interact with it
Note: this call requires an API key with admin rights
Note: this call is not available to Dataiku Cloud users
:param str login: the login of the new user
:param str password: the password of the new user
:param str display_name: the displayed name for the new user
:param str source_type: the type of new user. Admissible values are 'LOCAL' or 'LDAP'
:param list groups: the names of the groups the new user belongs to (defaults to `[]`)
:param str profile: The profile for the new user. Typical values (depend on your license): FULL_DESIGNER, DATA_DESIGNER, AI_CONSUMER, ...
:param str email: The email for the new user.
:return: A :class:`dataikuapi.dss.admin.DSSUser` user handle
"""
if groups is None:
groups = []
resp = self._perform_text(
"POST", "/admin/users/", body={
"login" : login,
"password" : password,
"displayName" : display_name,
"sourceType" : source_type,
"groups" : groups,
"userProfile" : profile,
"email": email
})
return DSSUser(self, login)
def create_users(self, users):
"""
Create multiple users, and return a list of creation status
Note: this call requires an API key with admin rights
Note: this call is not available to Dataiku Cloud users
:param list users: a list of dictionaries where each dictionary contains the parameters for user creation. It should contain the following keys:
- 'login' (str): the login of the new user
- 'password' (str): the password of the new user
- 'displayName' (str): the displayed name for the new user
- 'sourceType' (str): the type of new user. Admissible values are 'LOCAL' or 'LDAP'. Defaults to 'LOCAL'
- 'groups' (list): the names of the groups the new user belongs to
- 'userProfile' (str): The profile for the new user. Typical values (depend on your license): FULL_DESIGNER, DATA_DESIGNER, AI_CONSUMER, ... Defaults to 'DATA_SCIENTIST'
- 'email' (str): The email for the new user. Defaults to None
:rtype: list[dict]
:return: A list of dictionaries, where each dictionary represents the creation status of a user. It should contain the following keys:
- 'login' (str): the login of the created user
- 'status' (str): the creation status of the user. Can be 'SUCCESS' or 'FAILURE'
- 'error' (str): the error that occurred during that user's creation. Empty if status is not 'FAILURE'.
"""
for user in users:
user.setdefault('login', '')
user.setdefault('displayName', '')
user.setdefault('sourceType', 'LOCAL')
user.setdefault('groups', [])
user.setdefault('userProfile', 'DATA_SCIENTIST')
user.setdefault('email', None)
if user['groups'] is None:
user['groups'] = []
response = self._perform_text("POST", "/admin/users/actions/bulk", body=users)
user_statuses = json.loads(response)
return user_statuses
def edit_users(self, user_changes):
"""
Edits multiple users in a single bulk operation. This method is very permissive and is intended
for mass operations. If you are modifying a small number of users, it is advised to get a handle
from the get_user method and interact directly with a `DSSUser` object.
A valid workflow is to get the full users dictionaries from `list_users(include_settings=True)`,
modify them and use this method to apply the modifications.
Note: This call requires an API key with admin rights.
Note: this call is not available to Dataiku Cloud users
:param list[dict] user_changes: A list of dictionaries, where each dictionary defines the
changes for a single user. Each dictionary **must** contain the
'login' key to identify the user. Other keys can be included
to modify the user's properties, matching the structure of a
user settings object (see the output of
`list_users(include_settings=True))`. Available keys include:
- 'login' (str): The login of the user to modify (mandatory). Cannot be modified.
- 'displayName' (str): The user's display name.
- 'email' (str): The user's email address.
- 'groups' (list[str]): The list of groups for the user.
- 'userProfile' (str): The user's profile (e.g., 'FULL_DESIGNER').
- 'enabled' (bool): Whether the user is enabled.
- 'sourceType': User provisioning source type. Admissible values are 'LOCAL' or 'LDAP'.
- 'adminProperties' (dict): Custom admin properties for the user.
- 'userProperties' (dict): Custom user properties for the user.
- 'secrets' (list): A list of user-specific secrets.
- 'preferences' (dict): A dictionary of user preferences:
- 'uiLanguage' (str): UI language code (e.g., 'en', 'ja', 'fr').
- 'mentionEmails' (bool): Email notifications for mentions.
- 'discussionEmails' (bool): Email notifications for discussions.
- 'accessRequestEmails' (bool): Email notifications for access requests.
- 'grantedAccessEmails' (bool): Email notifications when access is granted.
- 'grantedPluginRequestEmails' (bool): Email notifications when a plugin request is granted.
- 'pluginRequestEmails' (bool): Email notifications for plugin requests.
- 'instanceAccessRequestsEmails' (bool): Email notifications for instance access requests.
- 'profileUpgradeRequestsEmails' (bool): Email notifications for profile upgrade requests.
- 'codeEnvCreationRequestEmails' (bool): Email notifications for code env creation requests.
- 'grantedCodeEnvCreationRequestEmails' (bool): Email notifications when a code env request is granted.
- 'dailyDigestsEmails' (bool): Daily digest emails.
- 'offlineActivityEmails' (bool): Offline activity summary emails.
- 'rememberPositionFlow' (bool): Remember position in the Flow.
- 'loginLogoutNotifications' (bool): Notifications for login/logout.
- 'watchedObjectsEditionsNotifications' (bool): Notifications for edits on watched objects.
- 'objectOnCurrentProjectCreatedDeletedNotifications' (bool): Notifications for object creation/deletion in the current project.
- 'anyObjectOnCurrentProjectEditedNotifications' (bool): Notifications for any object edit in the current project.
- 'watchStarOnCurrentProjectNotifications' (bool): Notifications for watch/star actions in the current project.
- 'otherUsersJobsTasksNotifications' (bool): Notifications for jobs/tasks from other users.
- 'requestAccessNotifications' (bool): Notifications for access requests.
- 'scenarioRunNotifications' (bool): Notifications for scenario runs.
:rtype: list[dict]
:return: A list of dictionaries, one for each attempted modification, indicating the status. Each dictionary contains the following keys:
- 'login' (str): The login of the user that was modified.
- 'status' (str): The result of the operation, either 'SUCCESS' or 'FAILURE'.
- 'error' (str): The error message if the status is 'FAILURE', otherwise empty.
"""
response = self._perform_text("PUT", "/admin/users/actions/bulk", body=user_changes)
user_statuses = json.loads(response)
return user_statuses
def delete_users(self, user_logins, allow_self_deletion=False):
"""
Bulk deletes multiple users.
Note: This call requires an API key with admin rights.
Note: this call is not available to Dataiku Cloud users
:param list[str] user_logins: A list of logins for the users to be deleted.
:param bool allow_self_deletion: Allow the use of this function to delete your own user. Warning: this is very dangerous and used recklessly could lead to the deletion of all users/admins.
:rtype: list[dict]
:return: A list of dictionaries, one for each attempted deletion, indicating the status. Each dictionary contains the following keys:
- 'login' (str): The login of the user that was deleted.
- 'status' (str): The result of the deletion, either 'SUCCESS' or 'FAILURE'.
- 'error' (str): The error message if the status is 'FAILURE', otherwise empty.
"""
params = {
'allowSelfDeletion': allow_self_deletion
}
response = self._perform_text("DELETE", "/admin/users/actions/bulk", body=user_logins, params=params)
user_statuses = json.loads(response)
return user_statuses
def get_own_user(self):
"""
Get a handle to interact with the current user
:return: A :class:`dataikuapi.dss.admin.DSSOwnUser` user handle
"""
return DSSOwnUser(self)
def list_users_activity(self, enabled_users_only=False):
"""
List all users activity
Note: this call requires an API key with admin rights
:return: A list of user activity logs, as a list of :class:`dataikuapi.dss.admin.DSSUserActivity` if as_objects is True, else as a list of dict
:rtype: list of :class:`dataikuapi.dss.admin.DSSUserActivity` or a list of dict
"""
params = {
"enabledUsersOnly": enabled_users_only
}
all_activity = self._perform_json("GET", "/admin/users-activity", params=params)
return [DSSUserActivity(self, user_activity["login"], user_activity) for user_activity in all_activity]
def list_expired_trial_users(self):
"""
List users whose trials have expired
:return: A list of users
"""
users = self._perform_json("GET", "/admin/list-trial-expired-users")
return users
def get_authorization_matrix(self):
"""
Get the authorization matrix for all enabled users and groups
Note: this call requires an API key with admin rights
:return: The authorization matrix
:rtype: A :class:`dataikuapi.dss.admin.DSSAuthorizationMatrix` authorization matrix handle
"""
resp = self._perform_json("GET", "/admin/authorization-matrix")
return DSSAuthorizationMatrix(resp)
def start_resync_users_from_supplier(self, logins):
"""
Starts a resync of multiple users from an external supplier (LDAP, Azure AD or custom auth)
:param list logins: list of logins to resync
:return: a :class:`dataikuapi.dss.future.DSSFuture` representing the sync process
:rtype: :class:`dataikuapi.dss.future.DSSFuture`
"""
future_resp = self._perform_json("POST", "/admin/users/actions/resync-multi", body=logins)
return DSSFuture.from_resp(self, future_resp)
def start_resync_all_users_from_supplier(self):
"""
Starts a resync of all users from an external supplier (LDAP, Azure AD or custom auth)
:return: a :class:`dataikuapi.dss.future.DSSFuture` representing the sync process
:rtype: :class:`dataikuapi.dss.future.DSSFuture`
"""
future_resp = self._perform_json("POST", "/admin/users/actions/resync-multi")
return DSSFuture.from_resp(self, future_resp)
def start_fetch_external_groups(self, user_source_type):
"""
Fetch groups from external source
:param user_source_type: 'LDAP', 'AZURE_AD' or 'CUSTOM'
:rtype: :class:`dataikuapi.dss.future.DSSFuture`
:return: a DSSFuture containing a list of group names
"""
future_resp = self._perform_json("GET", "/admin/external-groups", params={'userSourceType': user_source_type})
return DSSFuture.from_resp(self, future_resp)
def start_fetch_external_users(self, user_source_type, login=None, email=None, group_name=None):
"""
Fetch users from external source filtered by login or group name:
- if login or email is provided, will search for a user with an exact match in the external source (e.g. before login remapping)
- else,
- if group_name is provided, will search for members of the group in the external source
- else will search for all users
:param user_source_type: 'LDAP', 'AZURE_AD' or 'CUSTOM'
:param login: optional - the login of the user in the external source
:param email: optional - the email of the user in the external source
:param group_name: optional - the group name of the group in the external source
:rtype: :class:`dataikuapi.dss.future.DSSFuture`
:return: a DSSFuture containing a list of ExternalUser
"""
future_resp = self._perform_json("GET", "/admin/external-users", params={'userSourceType': user_source_type, 'login': login, 'email': email, 'groupName': group_name})
return DSSFuture.from_resp(self, future_resp)
def start_provision_users(self, user_source_type, users):
"""
Provision users of given source type
:param string user_source_type: 'LDAP', 'AZURE_AD' or 'CUSTOM'
:param list users: list of user attributes coming form the external source
:rtype: :class:`dataikuapi.dss.future.DSSFuture`
"""
future_resp = self._perform_json("POST", "/admin/users/actions/provision", body={'userSourceType': user_source_type, 'users': users})
return DSSFuture.from_resp(self, future_resp)
########################################################
# Groups
########################################################
def list_groups(self):
"""
List all groups setup on the DSS instance
Note: this call requires an API key with admin rights
:returns: A list of groups, as an list of dicts
:rtype: list of dicts
"""
return self._perform_json(
"GET", "/admin/groups/")
def get_group(self, name):
"""
Get a handle to interact with a specific group
:param str name: the name of the desired group
:returns: A :class:`dataikuapi.dss.admin.DSSGroup` group handle
"""
return DSSGroup(self, name)
def create_group(self, name, description=None, source_type='LOCAL'):
"""
Create a group, and return a handle to interact with it
Note: this call requires an API key with admin rights
:param str name: the name of the new group
:param str description: (optional) a description of the new group
:param source_type: the type of the new group. Admissible values are 'LOCAL' and 'LDAP'
:returns: A :class:`dataikuapi.dss.admin.DSSGroup` group handle
"""
resp = self._perform_text(
"POST", "/admin/groups/", body={
"name" : name,
"description" : description,
"sourceType" : source_type
})
return DSSGroup(self, name)
########################################################
# Connections
########################################################
def list_connections(self, as_type="dictitems"):
"""
List all connections setup on the DSS instance.
.. note::
This call requires an API key with admin rights
:param string as_type: how to return the connection. Possible values are "dictitems", "listitems" and "objects"
:return: if **as_type** is dictitems, a dict of connection name to :class:`dataikuapi.dss.admin.DSSConnectionListItem`.
if **as_type** is listitems, a list of :class:`dataikuapi.dss.admin.DSSConnectionListItem`.
if **as_type** is objects, a list of :class:`dataikuapi.dss.admin.DSSConnection`.
:rtype: dict or list
"""
items_dict = self._perform_json(
"GET", "/admin/connections/")
if as_type == "dictitems" or as_type == "dictitem":
return {name:DSSConnectionListItem(self, items_dict[name]) for name in items_dict.keys()}
if as_type == "listitems" or as_type == "listitem":
return [DSSConnectionListItem(self, items_dict[name]) for name in items_dict.keys()]
elif as_type == "objects" or as_type == "object":
return [DSSConnection(self, name) for name in items_dict.keys()]
else:
raise ValueError("Unknown as_type")
def list_connections_names(self, connection_type):
"""
List all connections names on the DSS instance.
:param str connection_type: Returns only connections with this type. Use 'all' if you don't want to filter.
:return: the list of connections names
:rtype: List[str]
"""
return self._perform_json("GET", "/connections/get-names", params={"type": connection_type})
def get_connection(self, name):
"""
Get a handle to interact with a specific connection
:param str name: the name of the desired connection
:returns: A :class:`dataikuapi.dss.admin.DSSConnection` connection handle
"""
return DSSConnection(self, name)
def create_connection(self, name, type, params=None, usable_by='ALL', allowed_groups=None, description=None):
"""
Create a connection, and return a handle to interact with it
Note: this call requires an API key with admin rights
:param name: the name of the new connection
:param type: the type of the new connection
:param dict params: the parameters of the new connection, as a JSON object (defaults to `{}`)
:param usable_by: the type of access control for the connection. Either 'ALL' (=no access control)
or 'ALLOWED' (=access restricted to users of a list of groups)
:param list allowed_groups: when using access control (that is, setting usable_by='ALLOWED'), the list
of names of the groups whose users are allowed to use the new connection (defaults to `[]`)
:param str description: (optional) a description of the new connection
:returns: A :class:`dataikuapi.dss.admin.DSSConnection` connection handle
"""
if params is None:
params = {}
if allowed_groups is None:
allowed_groups = []
resp = self._perform_text(
"POST", "/admin/connections/", body={
"name" : name,
"description" : description,
"type" : type,
"params" : params,
"usableBy" : usable_by,
"allowedGroups" : allowed_groups
})
return DSSConnection(self, name)
########################################################
# Code envs
########################################################
def list_code_envs(self, as_objects=False):
"""
List all code envs setup on the DSS instance
:param boolean as_objects: if True, each returned item will be a :class:`dataikuapi.dss.future.DSSCodeEnv`
:returns: a list of code envs. Each code env is a dict containing at least "name", "type" and "language"
"""
list = self._perform_json(
"GET", "/admin/code-envs/")
if as_objects:
return [DSSCodeEnv(self, e.get("envLang"), e.get("envName")) for e in list]
else:
return list
def get_code_env(self, env_lang, env_name):
"""
Get a handle to interact with a specific code env
:param env_lang: the language (PYTHON or R) of the new code env
:param env_name: the name of the new code env
:returns: A :class:`dataikuapi.dss.admin.DSSCodeEnv` code env handle
"""
return DSSCodeEnv(self, env_lang, env_name)
def create_internal_code_env(self, internal_env_type, python_interpreter=None, code_env_version=None, wait=True):
"""
Create a Python internal code environment, and return a handle to interact with it.
Note: this call requires an API key with `Create code envs` or `Manage all code envs` permission
Example:
.. code-block:: python
env_handle = client.create_internal_code_env(internal_env_type="RAG_CODE_ENV", python_interpreter="PYTHON310")
:param str internal_env_type: the internal env type, can be `DEEP_HUB_IMAGE_CLASSIFICATION_CODE_ENV`, `DEEP_HUB_IMAGE_OBJECT_DETECTION_CODE_ENV`, `PROXY_MODELS_CODE_ENV`, `DATABRICKS_UTILS_CODE_ENV`, `PII_DETECTION_CODE_ENV`, `HUGGINGFACE_LOCAL_CODE_ENV`, `RAG_CODE_ENV`, `DOCUMENT_EXTRACTION_CODE_ENV`, `DOCUMENT_TEMPLATING_CODE_ENV`.
:param str python_interpreter: Python interpreter version, can be `PYTHON39`, `PYTHON310`, `PYTHON311`, `PYTHON312` or `PYTHON313`. If None, DSS will try to select a supported & available interpreter.
:param str code_env_version: Version of the code env. Reserved for future use.
:param bool wait: wait for the code env to be created or return a future
:returns: A :class:`dataikuapi.dss.admin.DSSCodeEnv` code env handle or a `dataikuapi.dss.future.DSSFuture` if `wait` is False
"""
request_params = {
'dssInternalCodeEnvType': internal_env_type,
'pythonInterpreter': python_interpreter,
'codeEnvVersion': code_env_version,
'wait': wait
}
response = self._perform_json("POST", "/admin/code-envs/internal-env/create", params=request_params)
if response is None:
raise Exception('Env creation returned no data')
if response.get('messages', {}).get('error', False):
raise Exception('Env creation failed : %s' % (json.dumps(response.get('messages', {}).get('messages', {}))))
if not wait:
return DSSFuture.from_resp(self, response)
return DSSCodeEnv(self, "python", response["envName"])
def create_code_env(self, env_lang, env_name, deployment_mode, params=None, wait=True):
"""
Create a code env, and return a handle to interact with it
Note: this call requires an API key with `Create code envs` or `Manage all code envs` permission
:param env_lang: the language (PYTHON or R) of the new code env
:param env_name: the name of the new code env
:param deployment_mode: the type of the new code env
:param params: the parameters of the new code env, as a JSON object
:param bool wait: wait for the code env to be created or return a future
:returns: A :class:`dataikuapi.dss.admin.DSSCodeEnv` code env handle
"""
body_params = params if params is not None else {}
body_params['deploymentMode'] = deployment_mode